diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-07-09 13:52:30 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-07-09 13:52:30 +0200 |
commit | 273e9572331ae4360d3f0e4e7089dfc86392983d (patch) | |
tree | 3b0b1b46a4e793ac10c3ca0613f21c33ac175df7 | |
parent | 95a7a658c876c62de0f04226f80f5ce5003e77a4 (diff) | |
parent | 6422dedc3926d604e071c087e0e8263f4c8f9390 (diff) |
Merge branch 'master' into musum/cleanup-ConfigCurator-1
215 files changed, 2736 insertions, 1810 deletions
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java index 63347c2475a..29f895ce176 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java @@ -35,7 +35,7 @@ import static org.junit.Assert.fail; public class ContainerTest { @Test - public void jdisc_can_be_used_as_top_level_element() { + public void container_can_be_used_as_top_level_element() { try (JDisc container = fromServicesXml("<container version=\"1.0\">" + // "<search />" + // "</container>", Networking.disable)) { @@ -44,7 +44,7 @@ public class ContainerTest { } @Test - public void jdisc_id_can_be_set() { + public void container_id_can_be_set() { try (JDisc container = fromServicesXml("<container version=\"1.0\" id=\"my-service-id\">" + // "<search />" + // "</container>", Networking.disable)) { @@ -53,7 +53,7 @@ public class ContainerTest { } @Test - public void jdisc_can_be_embedded_in_services_tag() { + public void container_can_be_embedded_in_services_tag() { try (JDisc container = fromServicesXml("<services>" + // "<container version=\"1.0\" id=\"my-service-id\">" + // "<search />" + // @@ -65,7 +65,7 @@ public class ContainerTest { @Test @SuppressWarnings("try") // container is unused inside the try block - public void multiple_jdisc_elements_gives_exception() { + public void multiple_container_elements_gives_exception() { try (JDisc container = fromServicesXml("<services>" + // "<container version=\"1.0\" id=\"id1\" />" + // "<container version=\"1.0\" />" + // diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 855b3afafaf..3c5f96a1fec 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -28,6 +28,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -79,16 +85,6 @@ <scope>provided</scope> </dependency> - <!-- COMPILE --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <!-- TEST --> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml index 31e21eb18fd..f305d02c4ae 100644 --- a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml +++ b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml @@ -16,8 +16,7 @@ <process-resources>org.apache.maven.plugins:maven-resources-plugin:resources</process-resources> <compile>org.apache.maven.plugins:maven-compiler-plugin:compile</compile> <process-test-resources> - org.apache.maven.plugins:maven-resources-plugin:testResources, - com.yahoo.vespa:bundle-plugin:generate-bundle-classpath-mappings + org.apache.maven.plugins:maven-resources-plugin:testResources </process-test-resources> <test-compile>org.apache.maven.plugins:maven-compiler-plugin:testCompile</test-compile> <test>org.apache.maven.plugins:maven-surefire-plugin:test</test> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index af2168545dc..c173b2c11d9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -28,6 +28,7 @@ public class SummaryClass extends Derived { /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; private final boolean rawAsBase64; + private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); @@ -44,6 +45,7 @@ public class SummaryClass extends Derived { public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) { this.deployLogger = deployLogger; this.rawAsBase64 = search.isRawAsBase64(); + this.omitSummaryFeatures = summary.omitSummaryFeatures(); deriveName(summary); deriveFields(search,summary); deriveImplicitFields(summary); @@ -128,7 +130,8 @@ public class SummaryClass extends Derived { } classBuilder. id(id). - name(getName()); + name(getName()). + omitsummaryfeatures(omitSummaryFeatures); for (SummaryClassField field : fields.values() ) { classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). name(field.getName()). diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 3c6aa881af4..454d0f08ff0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -14,6 +14,7 @@ import java.util.List; public class DocumentSummary extends FieldView { private boolean fromDisk = false; + private boolean omitSummaryFeatures = false; private DocumentSummary inherited; /** @@ -30,6 +31,14 @@ public class DocumentSummary extends FieldView { /** Returns whether the user has noted explicitly that this summary accesses disk */ public boolean isFromDisk() { return fromDisk; } + public void setOmitSummaryFeatures(boolean value) { + omitSummaryFeatures = value; + } + + public boolean omitSummaryFeatures() { + return omitSummaryFeatures; + } + /** * The model is constrained to ensure that summary fields of the same name * in different classes have the same summary transform, because this is diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index fde3e3f012e..317ed0f66c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -111,8 +111,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); - addSimpleComponent("com.yahoo.container.jdisc.messagebus.SessionCache"); - addMetricsHandlers(); addTestrunnerComponentsIfTester(deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 37bfb8821c3..65247f29281 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -10,7 +10,9 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.xml.PlatformBundles; +import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -26,17 +28,21 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String BUNDLE_NAME = "model-evaluation"; + private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; private final static String REST_BINDING_PATH = "/model-evaluation/v1"; + public static final Path MODEL_EVALUATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(EVALUATION_BUNDLE_NAME); + public static final Path MODEL_INTEGRATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(INTEGRATION_BUNDLE_NAME); + /** Global rank profiles, aka models */ private final RankProfileList rankProfileList; public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); - cluster.addSimpleComponent(EVALUATOR_NAME, null, BUNDLE_NAME); + cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME); cluster.addComponent(ContainerModelEvaluation.getHandler()); } @@ -64,7 +70,7 @@ public class ContainerModelEvaluation implements } public static Handler<?> getHandler() { - Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); + Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); handler.addServerBindings( SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index a37d0ef416f..a1f52cca9fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; @@ -61,6 +62,11 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); } + @Override + protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + // Model evaluation bundles are pre-installed in the standalone container. + } + /** Note: using {@link CloudConfigOptions} as {@link DeployState#isHosted()} returns <em>false</em> for hosted configserver/controller */ private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3b04d536300..5bf8aa5228e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -187,6 +187,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { throwUponRestApi(spec); // TODO: remove addServlets(deployState, spec, cluster); addModelEvaluation(spec, cluster, context); + addModelEvaluationBundles(cluster); addProcessing(deployState, spec, cluster); addSearch(deployState, spec, cluster); @@ -565,6 +566,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles)); } + protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + /* These bundles are added to all application container clusters, even if they haven't + * declared 'model-evaluation' in services.xml, because there are many public API packages + * in the model-evaluation bundle that could be used by customer code. */ + cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE); + cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE); + } + private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index c247100db76..fb24b50aa99 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -291,6 +291,7 @@ TOKEN : | < TO: "to" > | < DIRECT: "direct" > | < FROMDISK: "from-disk" > +| < OMITSUMMARYFEATURES: "omit-summary-features" > | < ALWAYS: "always" > | < ONDEMAND: "on-demand" > | < NEVER: "never" > @@ -1767,6 +1768,7 @@ Object documentSummary(Search search) : lbrace() ( <FROMDISK> { summary.setFromDisk(true); } | + <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } | documentSummaryItem(summary) | <NL> )* diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg index a56d0e6aa4f..f497461b460 100644 --- a/config-model/src/test/derived/advanced/summary.cfg +++ b/config-model/src/test/derived/advanced/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1271952241 classes[].id 1271952241 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "debug" classes[].fields[].type "longstring" classes[].fields[].name "attributes" @@ -25,6 +26,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 472092010 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "location_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index c1679c57d1a..965c875d5ce 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 252850086 classes[].id 252850086 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "elem_array" classes[].fields[].type "jsonstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg index 13b17464946..f0189f9a3c7 100644 --- a/config-model/src/test/derived/attributeprefetch/summary.cfg +++ b/config-model/src/test/derived/attributeprefetch/summary.cfg @@ -1,27 +1,29 @@ defaultsummaryid 1151071433 -classes[0].id 1151071433 -classes[0].name "default" -classes[0].fields[0].name "rankfeatures" -classes[0].fields[0].type "featuredata" -classes[0].fields[1].name "summaryfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "documentid" -classes[0].fields[2].type "longstring" -classes[1].id 1980470965 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "singlebyte" -classes[1].fields[0].type "byte" -classes[1].fields[1].name "singleint" -classes[1].fields[1].type "integer" -classes[1].fields[2].name "singlelong" -classes[1].fields[2].type "int64" -classes[1].fields[3].name "singlefloat" -classes[1].fields[3].type "float" -classes[1].fields[4].name "singledouble" -classes[1].fields[4].type "double" -classes[1].fields[5].name "singlestring" -classes[1].fields[5].type "longstring" -classes[1].fields[6].name "rankfeatures" -classes[1].fields[6].type "featuredata" -classes[1].fields[7].name "summaryfeatures" -classes[1].fields[7].type "featuredata"
\ No newline at end of file +classes[].id 1151071433 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 1980470965 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "singlebyte" +classes[].fields[].type "byte" +classes[].fields[].name "singleint" +classes[].fields[].type "integer" +classes[].fields[].name "singlelong" +classes[].fields[].type "int64" +classes[].fields[].name "singlefloat" +classes[].fields[].type "float" +classes[].fields[].name "singledouble" +classes[].fields[].type "double" +classes[].fields[].name "singlestring" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg index 5bb472b5f41..2dac4736d23 100644 --- a/config-model/src/test/derived/complex/summary.cfg +++ b/config-model/src/test/derived/complex/summary.cfg @@ -1,41 +1,43 @@ defaultsummaryid 1506848752 -classes[0].id 1506848752 -classes[0].name "default" -classes[0].fields[0].name "woe" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "exact" -classes[0].fields[1].type "longstring" -classes[0].fields[2].name "title" -classes[0].fields[2].type "longstring" -classes[0].fields[3].name "dyntitle" -classes[0].fields[3].type "longstring" -classes[0].fields[4].name "source" -classes[0].fields[4].type "longstring" -classes[0].fields[5].name "stringfield" -classes[0].fields[5].type "longstring" -classes[0].fields[6].name "rankfeatures" -classes[0].fields[6].type "featuredata" -classes[0].fields[7].name "summaryfeatures" -classes[0].fields[7].type "featuredata" -classes[0].fields[8].name "documentid" -classes[0].fields[8].type "longstring" -classes[1].id 128090024 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "year_sub" -classes[1].fields[0].type "integer" -classes[1].fields[1].name "prefixenabled" -classes[1].fields[1].type "longstring" -classes[1].fields[2].name "fleeting2" -classes[1].fields[2].type "float" -classes[1].fields[3].name "foundat" -classes[1].fields[3].type "int64" -classes[1].fields[4].name "collapseby" -classes[1].fields[4].type "integer" -classes[1].fields[5].name "ts" -classes[1].fields[5].type "int64" -classes[1].fields[6].name "combineda" -classes[1].fields[6].type "integer" -classes[1].fields[7].name "rankfeatures" -classes[1].fields[7].type "featuredata" -classes[1].fields[8].name "summaryfeatures" -classes[1].fields[8].type "featuredata"
\ No newline at end of file +classes[].id 1506848752 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "woe" +classes[].fields[].type "longstring" +classes[].fields[].name "exact" +classes[].fields[].type "longstring" +classes[].fields[].name "title" +classes[].fields[].type "longstring" +classes[].fields[].name "dyntitle" +classes[].fields[].type "longstring" +classes[].fields[].name "source" +classes[].fields[].type "longstring" +classes[].fields[].name "stringfield" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 128090024 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "year_sub" +classes[].fields[].type "integer" +classes[].fields[].name "prefixenabled" +classes[].fields[].type "longstring" +classes[].fields[].name "fleeting2" +classes[].fields[].type "float" +classes[].fields[].name "foundat" +classes[].fields[].type "int64" +classes[].fields[].name "collapseby" +classes[].fields[].type "integer" +classes[].fields[].name "ts" +classes[].fields[].type "int64" +classes[].fields[].name "combineda" +classes[].fields[].type "integer" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg index ed3a61a5de5..82bed7fd55e 100644 --- a/config-model/src/test/derived/emptychild/summary.cfg +++ b/config-model/src/test/derived/emptychild/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1814603381 classes[].id 1814603381 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1490368133 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "a1" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg index e47f24b21c3..61294d97b4c 100644 --- a/config-model/src/test/derived/emptydefault/summary.cfg +++ b/config-model/src/test/derived/emptydefault/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1151071433 classes[].id 1151071433 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg index dbc9a90ebce..b50b970afe2 100644 --- a/config-model/src/test/derived/id/summary.cfg +++ b/config-model/src/test/derived/id/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1814716401 classes[].id 1814716401 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "uri" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg index 8da2598a5fc..3ab8e7e29e5 100644 --- a/config-model/src/test/derived/imported_position_field/summary.cfg +++ b/config-model/src/test/derived/imported_position_field/summary.cfg @@ -1,17 +1,19 @@ defaultsummaryid 1570252291 -classes[0].id 1570252291 -classes[0].name "default" -classes[0].fields[0].name "parent_ref" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "rankfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "summaryfeatures" -classes[0].fields[2].type "featuredata" -classes[0].fields[3].name "documentid" -classes[0].fields[3].type "longstring" -classes[1].id 1274088866 -classes[1].name "attributeprefetch" -classes[1].fields[0].name "rankfeatures" -classes[1].fields[0].type "featuredata" -classes[1].fields[1].name "summaryfeatures" -classes[1].fields[1].type "featuredata"
\ No newline at end of file +classes[].id 1570252291 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "parent_ref" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 1274088866 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg index b607786ca36..76faac23170 100644 --- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1194448774 classes[].id 1194448774 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "parent_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 890647799 classes[].name "mysummary" +classes[].omitsummaryfeatures false classes[].fields[].name "my_pos" classes[].fields[].type "jsonstring" classes[].fields[].name "rankfeatures" @@ -29,6 +31,7 @@ classes[].fields[].name "my_pos.distance" classes[].fields[].type "integer" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg index 3a9bf4f5e0a..ab6c6853925 100644 --- a/config-model/src/test/derived/imported_struct_fields/summary.cfg +++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg @@ -1,43 +1,47 @@ defaultsummaryid 1570252291 -classes[0].id 1570252291 -classes[0].name "default" -classes[0].fields[0].name "parent_ref" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "rankfeatures" -classes[0].fields[1].type "featuredata" -classes[0].fields[2].name "summaryfeatures" -classes[0].fields[2].type "featuredata" -classes[0].fields[3].name "documentid" -classes[0].fields[3].type "longstring" -classes[1].id 2126652894 -classes[1].name "mysummary" -classes[1].fields[0].name "documentid" -classes[1].fields[0].type "longstring" -classes[1].fields[1].name "my_elem_array" -classes[1].fields[1].type "jsonstring" -classes[1].fields[2].name "my_elem_map" -classes[1].fields[2].type "jsonstring" -classes[1].fields[3].name "my_str_int_map" -classes[1].fields[3].type "jsonstring" -classes[1].fields[4].name "rankfeatures" -classes[1].fields[4].type "featuredata" -classes[1].fields[5].name "summaryfeatures" -classes[1].fields[5].type "featuredata" -classes[2].id 1629947863 -classes[2].name "filtered" -classes[2].fields[0].name "elem_array_filtered" -classes[2].fields[0].type "jsonstring" -classes[2].fields[1].name "elem_map_filtered" -classes[2].fields[1].type "jsonstring" -classes[2].fields[2].name "str_int_map_filtered" -classes[2].fields[2].type "jsonstring" -classes[2].fields[3].name "rankfeatures" -classes[2].fields[3].type "featuredata" -classes[2].fields[4].name "summaryfeatures" -classes[2].fields[4].type "featuredata" -classes[3].id 1274088866 -classes[3].name "attributeprefetch" -classes[3].fields[0].name "rankfeatures" -classes[3].fields[0].type "featuredata" -classes[3].fields[1].name "summaryfeatures" -classes[3].fields[1].type "featuredata"
\ No newline at end of file +classes[].id 1570252291 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "parent_ref" +classes[].fields[].type "longstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].id 2126652894 +classes[].name "mysummary" +classes[].omitsummaryfeatures false +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].fields[].name "my_elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "my_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "my_str_int_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1629947863 +classes[].name "filtered" +classes[].omitsummaryfeatures false +classes[].fields[].name "elem_array_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "elem_map_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "str_int_map_filtered" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].id 1274088866 +classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg index f95949cfa62..74b5b44214e 100644 --- a/config-model/src/test/derived/importedfields/summary.cfg +++ b/config-model/src/test/derived/importedfields/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1294344677 classes[].id 1294344677 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "b_ref_with_summary" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 159551552 classes[].name "mysummary" +classes[].omitsummaryfeatures false classes[].fields[].name "a_ref" classes[].fields[].type "longstring" classes[].fields[].name "b_ref_with_summary" @@ -33,6 +35,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg index 5bdc8fcdef4..d04bc4eb167 100644 --- a/config-model/src/test/derived/indexswitches/summary.cfg +++ b/config-model/src/test/derived/indexswitches/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1698765342 classes[].id 1698765342 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "source" classes[].fields[].type "longstring" classes[].fields[].name "title" diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg index dde71a1378c..dde9f95ecbe 100644 --- a/config-model/src/test/derived/inheritance/summary.cfg +++ b/config-model/src/test/derived/inheritance/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1797992819 classes[].id 1797992819 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "onlyfather" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1608562186 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "onlygrandparent" classes[].fields[].type "integer" classes[].fields[].name "overridden" diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg index 267585e3fda..d5eb316ff01 100644 --- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg +++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1195656216 classes[].id 1195656216 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" classes[].fields[].name "artist" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1706878063 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "attinx" classes[].fields[].type "integer" classes[].fields[].name "artist" diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg index 24d6cab7697..b465bdfa541 100644 --- a/config-model/src/test/derived/map_attribute/summary.cfg +++ b/config-model/src/test/derived/map_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1376056200 classes[].id 1376056200 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "str_map" classes[].fields[].type "jsonstring" classes[].fields[].name "int_map" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index f70025c8f02..67988dbf30e 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1131098132 classes[].id 1131098132 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "str_elem_map" classes[].fields[].type "jsonstring" classes[].fields[].name "int_elem_map" diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg index cb5bf17df84..b6a53a9a1d9 100644 --- a/config-model/src/test/derived/mlr/summary.cfg +++ b/config-model/src/test/derived/mlr/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1868876861 classes[].id 1868876861 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" classes[].fields[].name "b" @@ -13,6 +14,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1944325986 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "a" classes[].fields[].type "longstring" classes[].fields[].name "ranklog" diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg index 3eca077dbc8..bc55727b407 100644 --- a/config-model/src/test/derived/music/summary.cfg +++ b/config-model/src/test/derived/music/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 2086497905 classes[].id 2086497905 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" classes[].fields[].name "sales" @@ -79,6 +80,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 2060710706 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" classes[].fields[].name "pto" diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg index 7cd92c26e02..0b98b20c342 100644 --- a/config-model/src/test/derived/newrank/summary.cfg +++ b/config-model/src/test/derived/newrank/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 912980235 classes[].id 912980235 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "bgndata" classes[].fields[].type "longstring" classes[].fields[].name "sales" @@ -71,6 +72,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1606815285 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "sales" classes[].fields[].type "integer" classes[].fields[].name "pto" diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg index fad012393ef..4222e88cc2f 100644 --- a/config-model/src/test/derived/position_nosummary/summary.cfg +++ b/config-model/src/test/derived/position_nosummary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1727020212 classes[].id 1727020212 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "pos.position" classes[].fields[].type "xmlstring" classes[].fields[].name "pos.distance" @@ -13,6 +14,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1530141163 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg index af801f43cc0..f54066d865e 100644 --- a/config-model/src/test/derived/position_summary/summary.cfg +++ b/config-model/src/test/derived/position_summary/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 230670304 classes[].id 230670304 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "pos" classes[].fields[].type "jsonstring" classes[].fields[].name "pos.position" @@ -15,6 +16,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1530141163 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "pos_zcurve" classes[].fields[].type "int64" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg index 6e33bd4e567..9cc613107e0 100644 --- a/config-model/src/test/derived/predicate_attribute/summary.cfg +++ b/config-model/src/test/derived/predicate_attribute/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1391971216 classes[].id 1391971216 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "some_predicate_field" classes[].fields[].type "string" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg index f8b56baf8f2..4f417a848a7 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankexpression/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1753207254 classes[].id 1753207254 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "artist" classes[].fields[].type "longstring" classes[].fields[].name "title" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1736696699 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "year" classes[].fields[].type "integer" classes[].fields[].name "foo1" diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg index 9644eb878ea..49b668e9edf 100644 --- a/config-model/src/test/derived/ranktypes/summary.cfg +++ b/config-model/src/test/derived/ranktypes/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1567556360 classes[].id 1567556360 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "title" classes[].fields[].type "longstring" classes[].fields[].name "descr" diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg index 49037473d88..410bccff7b3 100644 --- a/config-model/src/test/derived/reference_fields/summary.cfg +++ b/config-model/src/test/derived/reference_fields/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1987541865 classes[].id 1987541865 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "campaign_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -11,6 +12,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 428144659 classes[].name "explicit_summary" +classes[].omitsummaryfeatures false classes[].fields[].name "yet_another_ref" classes[].fields[].type "longstring" classes[].fields[].name "rankfeatures" @@ -19,6 +21,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].id 1274088866 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg index 28f19e6fe25..655499a88be 100644 --- a/config-model/src/test/derived/streamingstruct/summary.cfg +++ b/config-model/src/test/derived/streamingstruct/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 569269436 classes[].id 569269436 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "coupleof" classes[].fields[].type "longstring" classes[].fields[].name "anothersummaryfield" @@ -41,6 +42,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 109252281 classes[].name "summ" +classes[].omitsummaryfeatures false classes[].fields[].name "snippet" classes[].fields[].type "longstring" classes[].fields[].name "snippet2" diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg index caa44931c6a..a52b34925dc 100644 --- a/config-model/src/test/derived/streamingstructdefault/summary.cfg +++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 718801936 classes[].id 718801936 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "sum1" classes[].fields[].type "longstring" classes[].fields[].name "f1" diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg index fb32eacbb4c..355cba0e561 100644 --- a/config-model/src/test/derived/tensor/summary.cfg +++ b/config-model/src/test/derived/tensor/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 898020074 classes[].id 898020074 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "f1" classes[].fields[].type "tensor" classes[].fields[].name "f3" @@ -17,6 +18,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1476352352 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "f2" classes[].fields[].type "tensor" classes[].fields[].name "f3" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg new file mode 100644 index 00000000000..ead74110db3 --- /dev/null +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "tokenization" +ilscript[].docfield[] "text" +ilscript[].docfield[] "text_array" +ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }" +ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }" +ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }" +ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }"
\ No newline at end of file diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd new file mode 100644 index 00000000000..4510a574d60 --- /dev/null +++ b/config-model/src/test/derived/tokenization/tokenization.sd @@ -0,0 +1,23 @@ +schema tokenization { + + document tokenization { + + field text type string { + indexing: index | summary + } + + field text_array type array<string> { + indexing: index | summary + } + + } + + field text_derived type string { + indexing: input text | normalize | index | summary + } + + field text_array_derived type array<string> { + indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg index e5485a24c8c..e0e67a5669d 100644 --- a/config-model/src/test/derived/types/summary.cfg +++ b/config-model/src/test/derived/types/summary.cfg @@ -1,6 +1,7 @@ defaultsummaryid 1131946680 classes[].id 1131946680 classes[].name "default" +classes[].omitsummaryfeatures false classes[].fields[].name "abyte" classes[].fields[].type "byte" classes[].fields[].name "along" @@ -25,6 +26,7 @@ classes[].fields[].name "documentid" classes[].fields[].type "longstring" classes[].id 1027812395 classes[].name "attributeprefetch" +classes[].omitsummaryfeatures false classes[].fields[].name "other" classes[].fields[].type "int64" classes[].fields[].name "abyte" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index bfc738a4f87..ef7da4f23d0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.SummaryConfig; import org.junit.Test; import java.io.IOException; @@ -14,6 +15,7 @@ import java.util.Iterator; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Tests summary extraction @@ -151,4 +153,30 @@ public class SummaryTestCase extends SchemaTestCase { return builder.getSearch("ad"); } + @Test + public void omit_summary_features_specified_for_document_summary() throws ParseException { + String sd = joinLines( + "schema test {", + " document test {", + " field foo type string { indexing: summary }", + " }", + " document-summary bar {", + " summary foo type string {}", + " omit-summary-features", + " }", + " document-summary baz {", + " summary foo type string {}", + " }", + "}"); + var search = SearchBuilder.createFromString(sd).getSearch(); + assertOmitSummaryFeatures(true, search, "bar"); + assertOmitSummaryFeatures(false, search, "baz"); + } + + private void assertOmitSummaryFeatures(boolean expected, Search search, String summaryName) { + var summary = new SummaryClass(search, search.getSummary(summaryName), new BaseDeployLogger()); + var config = new SummaryConfig.Classes(summary.getSummaryClassConfig()); + assertEquals(expected, config.omitsummaryfeatures()); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java new file mode 100755 index 00000000000..6fe367ef6d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java @@ -0,0 +1,19 @@ +// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseh + */ +public class TokenizationTestCase extends AbstractExportingTestCase { + + @Test + public void testTokenizationScripts() throws IOException, ParseException { + assertCorrectDeriving("tokenization"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 2b55d7a3948..2144c3c9a66 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.text.XML; @@ -16,6 +17,7 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; import org.junit.Test; @@ -27,6 +29,8 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -124,6 +128,14 @@ public class ConfigserverClusterTest { assertTrue(config.zookeeperLocalhostAffinity()); } + @Test + public void model_evaluation_bundles_are_not_installed_via_config() { + // These bundles must be pre-installed because they are used by config-model. + PlatformBundlesConfig config = getConfig(PlatformBundlesConfig.class); + assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString()))); + assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString()))); + } + @SuppressWarnings("varargs") private static <T> void assertZookeeperServerProperty( List<ZookeeperServerConfig.Server> zkServers, Function<ZookeeperServerConfig.Server, T> propertyMapper, T... expectedProperties) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 543318f9224..ccee21c87dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -27,6 +27,7 @@ import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.VipStatusHandler; import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; @@ -52,6 +53,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.http.ConnectorFactory; @@ -124,6 +126,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { public TemporaryFolder applicationFolder = new TemporaryFolder(); @Test + public void model_evaluation_bundles_are_deployed() { + createBasicContainerModel(); + PlatformBundlesConfig config = root.getConfig(PlatformBundlesConfig.class, "default"); + assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())); + assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())); + } + + @Test public void deprecated_jdisc_tag_is_allowed() { Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", @@ -244,10 +254,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void verify_bindings_for_builtin_handlers() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' />" - ); - createModel(root, clusterElem); + createBasicContainerModel(); JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); JdiscBindingsConfig.Handlers defaultRootHandler = config.handlers(BindingsOverviewHandler.class.getName()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index 9e02572737e..7034176da14 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockRoot; import com.yahoo.container.ComponentsConfig; @@ -51,6 +52,11 @@ public abstract class ContainerModelBuilderTestBase { protected MockRoot root; + protected void createBasicContainerModel() { + Element clusterElem = DomBuilderTest.parse("<container id='default' version='1.0' />"); + createModel(root, clusterElem); + } + public static void createModel(MockRoot root, DeployState deployState, VespaModel vespaModel, Element... containerElems) { for (Element containerElem : containerElems) { ContainerModel model = new ContainerModelBuilder(false, ContainerModelBuilder.Networking.enable) diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index 29bd38ea891..628e1c013e6 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -11,8 +11,8 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.RawConfig; -import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -32,24 +32,29 @@ public class ConfigProxyRpcServerTest { private static final String hostname = "localhost"; private static final int port = 12345; private static final String configSourceAddress = "tcp/" + hostname + ":" + port; - private TestServer server; - private TestClient client; + private static TestServer server; + private static TestClient client; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void setup() throws ListenFailedException { + @BeforeClass + public static void setup() throws ListenFailedException { server = new TestServer(); client = new TestClient(server.listenPort()); } - @After - public void teardown() { + @AfterClass + public static void teardown() { client.close(); server.close(); } + private static void reset() throws ListenFailedException { + teardown(); + setup(); + } + @Test public void basic() { ProxyServer proxy = createTestServer(new MockConfigSource()); @@ -75,7 +80,9 @@ public class ConfigProxyRpcServerTest { * Tests listCachedConfig RPC command */ @Test - public void testRpcMethodListCachedConfig() { + public void testRpcMethodListCachedConfig() throws ListenFailedException { + reset(); + Request req = new Request("listCachedConfig"); client.invoke(req); @@ -129,7 +136,9 @@ public class ConfigProxyRpcServerTest { * Tests printStatistics RPC command */ @Test - public void testRpcMethodListSourceConnections() { + public void testRpcMethodListSourceConnections() throws ListenFailedException { + reset(); + Request req = new Request("listSourceConnections"); client.invoke(req); @@ -218,7 +227,9 @@ public class ConfigProxyRpcServerTest { * Tests updateSources RPC command */ @Test - public void testRpcMethodUpdateSources() { + public void testRpcMethodUpdateSources() throws ListenFailedException { + reset(); + Request req = new Request("updateSources"); String spec1 = "tcp/a:19070"; String spec2 = "tcp/b:19070"; diff --git a/config/pom.xml b/config/pom.xml index 6e4e26ed0f1..8355587c10b 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -15,6 +15,12 @@ <dependencies> <!-- provided scope --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def index 20ca6b10450..26b0e4a4a37 100644 --- a/configdefinitions/src/vespa/summary.def +++ b/configdefinitions/src/vespa/summary.def @@ -4,5 +4,6 @@ namespace=vespa.config.search defaultsummaryid int default=-1 classes[].id int classes[].name string +classes[].omitsummaryfeatures bool default=false classes[].fields[].name string classes[].fields[].type string diff --git a/configserver-client/pom.xml b/configserver-client/pom.xml index 0a29ba003f4..39005c9ccab 100644 --- a/configserver-client/pom.xml +++ b/configserver-client/pom.xml @@ -30,6 +30,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver/pom.xml b/configserver/pom.xml index 3b7fef085b1..a237d7e00ad 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -142,6 +142,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <!-- To get all necessary test deps. --> <groupId>com.yahoo.vespa</groupId> <artifactId>container-test</artifactId> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2d5940ebba2..56fd6a64305 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -56,9 +56,9 @@ import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SecretStoreValidator; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.TesterClient; -import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse; import com.yahoo.vespa.config.server.http.v2.PrepareResult; -import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse; import com.yahoo.vespa.config.server.metrics.DeploymentMetricsRetriever; import com.yahoo.vespa.config.server.metrics.ProtonMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -547,15 +547,27 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { + public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, String pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. - String relativePath = "clustercontroller-status/" + pathSuffix; + String pathPrefix; + switch (serviceName) { + case "container-clustercontroller": { + pathPrefix = "clustercontroller-status/v1/"; + break; + } + case "distributor": + case "storagenode": { + pathPrefix = ""; + break; + } + default: + throw new NotFoundException("No status page for service: " + serviceName); + } - return httpProxy.get(getApplication(applicationId), hostName, - CLUSTERCONTROLLER_CONTAINER.serviceName, relativePath); + return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix + pathSuffix); } public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index ff70ce1e1b2..cdbd7378151 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -18,8 +18,8 @@ import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.http.v2.HttpConfigRequests; -import com.yahoo.vespa.config.server.http.v2.TenantRequest; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.TenantRequest; import com.yahoo.vespa.config.util.ConfigUtils; /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index dfbce72d4ba..7e8940d28b3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -14,9 +14,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareAndActivateResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.model.content.Content; import org.apache.hc.core5.http.ContentType; import org.eclipse.jetty.http.MultiPartFormInputStream; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 3634a6825a3..ec7b6616bb6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -6,47 +6,42 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; -import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.jdisc.application.UriPattern; -import com.yahoo.slime.Cursor; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; +import com.yahoo.restapi.Path; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationReindexing; -import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; -import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.http.v2.request.ApplicationContentRequest; +import com.yahoo.vespa.config.server.http.v2.response.ApplicationSuspendedResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeleteApplicationResponse; +import com.yahoo.vespa.config.server.http.v2.response.GetApplicationResponse; +import com.yahoo.vespa.config.server.http.v2.response.QuotaUsageResponse; +import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; -import java.net.URLDecoder; import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Stream; import static com.yahoo.yolean.Exceptions.uncheck; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.stream.Collectors.toList; /** * Operations on applications (delete, wait for config convergence, restart, application content etc.) @@ -55,28 +50,6 @@ import static java.util.stream.Collectors.toList; */ public class ApplicationHandler extends HttpHandler { - private static final List<UriPattern> URI_PATTERNS = Stream.of( - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindex", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindexing", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/validate-secret-store", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/quota", - "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", - "http://*/application/v2/tenant/*/application/*") - .map(UriPattern::new) - .collect(toList()); - private final Zone zone; private final ApplicationRepository applicationRepository; @@ -90,152 +63,135 @@ public class ApplicationHandler extends HttpHandler { } @Override - public HttpResponse handleDELETE(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - - if (isReindexingRequest(request)) { - applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false)); - return createMessageResponse("Reindexing disabled"); - } + public HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri()); + + if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return getApplicationResponse(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value())); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return getApplicationResponse(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/content/{*}")) return content(applicationId(path), path.getRest(), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest()); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge")) return listServiceConverge(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/{command}")) return testerRequest(applicationId(path), path.get("command"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path)); + return ErrorResponse.notFoundError("Nothing at " + path); + } - if (applicationRepository.delete(applicationId)) - return new DeleteApplicationResponse(Response.Status.OK, applicationId); + @Override + public HttpResponse handlePOST(HttpRequest request) { + Path path = new Path(request.getUri()); - return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found"); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/run/{suite}")) return testerStartTests(applicationId(path), path.get("suite"), request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/validate-secret-store")) return validateSecretStore(applicationId(path), request); + return ErrorResponse.notFoundError("Nothing at " + path); } @Override - public HttpResponse handleGET(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); - - if (isServiceConvergeRequest(request)) { - // Expects both hostname and port in the request (hostname:port) - String hostAndPort = getHostNameFromRequest(request); - return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), - timeout, getVespaVersionFromRequest(request)); - } + public HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri()); - if (isClusterControllerStatusRequest(request)) { - String hostName = getHostNameFromRequest(request); - String pathSuffix = URLDecoder.decode(getPathSuffix(request), UTF_8); - return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix); - } + if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return deleteApplication(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value())); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return deleteApplication(applicationId(path)); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return disableReindexing(applicationId(path)); + return ErrorResponse.notFoundError("Nothing at " + path); + } - if (isReindexingRequest(request)) { - return getReindexingStatus(applicationId); - } + private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) { + return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), + getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + } - if (isContentRequest(request)) { - long sessionId = applicationRepository.getSessionIdForApplication(applicationId); - String contentPath = getBindingMatch(request).group(7); - ApplicationFile applicationFile = - applicationRepository.getApplicationFileFromSession(applicationId.tenant(), - sessionId, - contentPath, - ContentRequest.getApplicationFileMode(request.getMethod())); - ApplicationContentRequest contentRequest = new ApplicationContentRequest(request, - sessionId, - applicationId, - zone, - contentPath, - applicationFile); - return new ContentHandler().get(contentRequest); - } + private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) { + return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), + getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + } - if (isServiceConvergeListRequest(request)) { - return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), timeout, - getVespaVersionFromRequest(request)); - } + private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, String pathSuffix) { + return applicationRepository.serviceStatusPage(applicationId, hostname, service, pathSuffix); + } - if (isFiledistributionStatusRequest(request)) { - return applicationRepository.filedistributionStatus(applicationId, timeout); - } + private HttpResponse content(ApplicationId applicationId, String contentPath, HttpRequest request) { + long sessionId = applicationRepository.getSessionIdForApplication(applicationId); + ApplicationFile applicationFile = + applicationRepository.getApplicationFileFromSession(applicationId.tenant(), + sessionId, + contentPath, + ContentRequest.getApplicationFileMode(request.getMethod())); + ApplicationContentRequest contentRequest = new ApplicationContentRequest(request, + sessionId, + applicationId, + zone, + contentPath, + applicationFile); + return new ContentHandler().get(contentRequest); + } - if (isLogRequest(request)) { - Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); - String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse(""); - return applicationRepository.getLogs(applicationId, hostname, apiParams); - } + private HttpResponse filedistributionStatus(ApplicationId applicationId, HttpRequest request) { + return applicationRepository.filedistributionStatus(applicationId, getTimeoutFromRequest(request)); + } - if (isProtonMetricsRequest(request)) { - return applicationRepository.getProtonMetrics(applicationId); - } + private HttpResponse logs(ApplicationId applicationId, HttpRequest request) { + Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); + String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse(""); + return applicationRepository.getLogs(applicationId, hostname, apiParams); + } - if (isDeploymentMetricsRequest(request)) { - return applicationRepository.getDeploymentMetrics(applicationId); - } + private HttpResponse protonMetrics(ApplicationId applicationId) { + return applicationRepository.getProtonMetrics(applicationId); + } - if (isIsSuspendedRequest(request)) { - return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); - } + private HttpResponse deploymentMetrics(ApplicationId applicationId) { + return applicationRepository.getDeploymentMetrics(applicationId); + } - if (isTesterRequest(request)) { - String testerCommand = getTesterCommandFromRequest(request); - switch (testerCommand) { - case "status": - return applicationRepository.getTesterStatus(applicationId); - case "log": - Long after = Long.valueOf(request.getProperty("after")); - return applicationRepository.getTesterLog(applicationId, after); - case "ready": - return applicationRepository.isTesterReady(applicationId); - case "report": - return applicationRepository.getTestReport(applicationId); - default: - throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); - } - } + private HttpResponse isSuspended(ApplicationId applicationId) { + return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId)); + } - if (isQuotaUsageRequest(request)) { - var quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId); - return new QuotaUsageResponse(quotaUsageRate); + private HttpResponse testerRequest(ApplicationId applicationId, String command, HttpRequest request) { + switch (command) { + case "status": + return applicationRepository.getTesterStatus(applicationId); + case "log": + Long after = Long.valueOf(request.getProperty("after")); + return applicationRepository.getTesterLog(applicationId, after); + case "ready": + return applicationRepository.isTesterReady(applicationId); + case "report": + return applicationRepository.getTestReport(applicationId); + default: + throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); } + } - return getApplicationResponse(applicationId); + private HttpResponse quotaUsage(ApplicationId applicationId) { + double quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId); + return new QuotaUsageResponse(quotaUsageRate); } - GetApplicationResponse getApplicationResponse(ApplicationId applicationId) { + private HttpResponse getApplicationResponse(ApplicationId applicationId) { return new GetApplicationResponse(Response.Status.OK, - applicationRepository.getApplicationGeneration(applicationId), - applicationRepository.getAllVersions(applicationId), - applicationRepository.getApplicationPackageReference(applicationId)); + applicationRepository.getApplicationGeneration(applicationId), + applicationRepository.getAllVersions(applicationId), + applicationRepository.getApplicationPackageReference(applicationId)); } - @Override - public HttpResponse handlePOST(HttpRequest request) { - ApplicationId applicationId = getApplicationIdFromRequest(request); - - if (isRestartRequest(request)) - return restart(request, applicationId); - - if (isTesterStartTestsRequest(request)) { - byte[] data; - try { - data = IOUtils.readBytes(request.getData(), 1024 * 1000); - } catch (IOException e) { - throw new IllegalArgumentException("Could not read data in request " + request); - } - return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data); - } - - if (isReindexRequest(request)) { - return triggerReindexing(request, applicationId); - } - - if (isReindexingRequest(request)) { - applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true)); - return createMessageResponse("Reindexing enabled"); - } - - if (isValidateSecretStoreRequest(request)) { - var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes())); - return applicationRepository.validateSecretStore(applicationId, zone.system(), slime); - } - - throw new NotFoundException("Illegal POST request '" + request.getUri() + "'"); + public HttpResponse deleteApplication(ApplicationId applicationId) { + if (applicationRepository.delete(applicationId)) + return new DeleteApplicationResponse(applicationId); + return ErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found"); } + private Model getActiveModelOrThrow(ApplicationId id) { return applicationRepository.getActiveApplicationSet(id) .orElseThrow(() -> new NotFoundException("Application '" + id + "' not found")) @@ -243,7 +199,7 @@ public class ApplicationHandler extends HttpHandler { .getModel(); } - private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) { + private HttpResponse triggerReindexing(ApplicationId applicationId, HttpRequest request) { Model model = getActiveModelOrThrow(applicationId); Map<String, Set<String>> documentTypes = model.documentTypesByCluster(); Map<String, Set<String>> indexedDocumentTypes = model.indexedDocumentTypesByCluster(); @@ -274,7 +230,7 @@ public class ApplicationHandler extends HttpHandler { return reindexing; }); - return createMessageResponse(reindexed.entrySet().stream() + return new MessageResponse(reindexed.entrySet().stream() .filter(cluster -> ! cluster.getValue().isEmpty()) .map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'") .reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId) @@ -284,6 +240,16 @@ public class ApplicationHandler extends HttpHandler { .toString()); } + public HttpResponse disableReindexing(ApplicationId applicationId) { + applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false)); + return new MessageResponse("Reindexing disabled"); + } + + private HttpResponse enableReindexing(ApplicationId applicationId) { + applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true)); + return new MessageResponse("Reindexing enabled"); + } + private HttpResponse getReindexingStatus(ApplicationId applicationId) { Tenant tenant = applicationRepository.getTenant(applicationId); if (tenant == null) @@ -294,239 +260,42 @@ public class ApplicationHandler extends HttpHandler { applicationRepository.getClusterReindexingStatus(applicationId)); } - private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { - if (getBindingMatch(request).groupCount() != 7) - throw new NotFoundException("Illegal POST restart request '" + request.getUri() + - "': Must have 6 arguments but had " + (getBindingMatch(request).groupCount() - 1)); - applicationRepository.restart(applicationId, hostFilterFrom(request)); - return new JSONResponse(Response.Status.OK); // return empty - } - - private HostFilter hostFilterFrom(HttpRequest request) { - return HostFilter.from(request.getProperty("hostname"), - request.getProperty("flavor"), - request.getProperty("clusterType"), - request.getProperty("clusterId")); - } - - private static BindingMatch<?> getBindingMatch(HttpRequest request) { - return URI_PATTERNS.stream() - .map(pattern -> { - UriPattern.Match match = pattern.match(request.getUri()); - if (match == null) return null; - return new BindingMatch<>(match, new Object(), pattern); - }) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Illegal url for config request: " + request.getUri())); - } - - private static boolean isRestartRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/restart"); - } - - private static boolean isReindexRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/reindex"); - } - - private static boolean isReindexingRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/reindexing"); - } - - private static boolean isIsSuspendedRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/suspended"); - } - - private static boolean isProtonMetricsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().endsWith("/metrics/proton"); - } - - private static boolean isDeploymentMetricsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().endsWith("/metrics/deployment"); - } - - private static boolean isLogRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/logs"); - } - - private static boolean isValidateSecretStoreRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/validate-secret-store"); - } - - private static boolean isServiceConvergeListRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/serviceconverge"); - } - - private static boolean isServiceConvergeRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().contains("/serviceconverge/"); - } - - private static boolean isClusterControllerStatusRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 9 && - request.getUri().getPath().contains("/clustercontroller/"); - } - - private static boolean isContentRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() > 7 && - request.getUri().getPath().contains("/content/"); - } - - private static boolean isFiledistributionStatusRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().contains("/filedistributionstatus"); - } - - private static boolean isTesterRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 8 && - request.getUri().getPath().contains("/tester"); - } - - private static boolean isTesterStartTestsRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 9 && - request.getUri().getPath().contains("/tester/run/"); - } - - private static boolean isQuotaUsageRequest(HttpRequest request) { - return getBindingMatch(request).groupCount() == 7 && - request.getUri().getPath().endsWith("/quota"); - } - - private static String getHostNameFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(7); - } - - private static String getTesterCommandFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(7); - } - - private static String getSuiteFromRequest(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(8); - } - - private static String getPathSuffix(HttpRequest req) { - BindingMatch<?> bm = getBindingMatch(req); - return bm.group(8); - } - - private static ApplicationId getApplicationIdFromRequest(HttpRequest req) { - // Two bindings for this: with full app id or only application name - BindingMatch<?> bm = getBindingMatch(req); - if (bm.groupCount() > 4) return createFromRequestFullAppId(bm); - return createFromRequestSimpleAppId(bm); - } - - // The URL pattern with only tenant and application given - private static ApplicationId createFromRequestSimpleAppId(BindingMatch<?> bm) { - TenantName tenant = TenantName.from(bm.group(2)); - ApplicationName application = ApplicationName.from(bm.group(3)); - return new ApplicationId.Builder().tenant(tenant).applicationName(application).build(); - } - - // The URL pattern with full app id given - private static ApplicationId createFromRequestFullAppId(BindingMatch<?> bm) { - String tenant = bm.group(2); - String application = bm.group(3); - String instance = bm.group(6); - return new ApplicationId.Builder() - .tenant(tenant) - .applicationName(application).instanceName(instance) - .build(); - } - - private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) { - String vespaVersion = request.getProperty("vespaVersion"); - return (vespaVersion == null || vespaVersion.isEmpty()) - ? Optional.empty() - : Optional.of(Version.fromString(vespaVersion)); + private HttpResponse restart(ApplicationId applicationId, HttpRequest request) { + HostFilter filter = HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId")); + applicationRepository.restart(applicationId, filter); + return new MessageResponse("Success"); } - private static class DeleteApplicationResponse extends JSONResponse { - DeleteApplicationResponse(int status, ApplicationId applicationId) { - super(status); - object.setString("message", "Application '" + applicationId + "' deleted"); + private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) { + byte[] data; + try { + data = IOUtils.readBytes(request.getData(), 1024 * 1000); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read data in request " + request); } + return applicationRepository.startTests(applicationId, suite, data); } - private static class GetApplicationResponse extends JSONResponse { - GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) { - super(status); - object.setLong("generation", generation); - object.setString("applicationPackageFileReference", applicationPackageReference.orElse("")); - Cursor modelVersionArray = object.setArray("modelVersions"); - modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); - } + private HttpResponse validateSecretStore(ApplicationId applicationId, HttpRequest request) { + var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes())); + return applicationRepository.validateSecretStore(applicationId, zone.system(), slime); } - private static class ApplicationSuspendedResponse extends JSONResponse { - ApplicationSuspendedResponse(boolean suspended) { - super(Response.Status.OK); - object.setBool("suspended", suspended); - } + private static ApplicationId applicationId(Path path) { + return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); } - private static class QuotaUsageResponse extends JSONResponse { - QuotaUsageResponse(double usageRate) { - super(Response.Status.OK); - object.setDouble("rate", usageRate); - } + private static Duration getTimeoutFromRequest(HttpRequest request) { + return HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); } - static class ReindexingResponse extends JSONResponse { - ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing, - Map<String, ClusterReindexing> clusters) { - super(Response.Status.OK); - object.setBool("enabled", reindexing.enabled()); - Cursor clustersObject = object.setObject("clusters"); - documentTypes.forEach((cluster, types) -> { - Cursor clusterObject = clustersObject.setObject(cluster); - Cursor pendingObject = clusterObject.setObject("pending"); - Cursor readyObject = clusterObject.setObject("ready"); - - for (String type : types) { - Cursor statusObject = readyObject.setObject(type); - if (reindexing.clusters().containsKey(cluster)) { - if (reindexing.clusters().get(cluster).pending().containsKey(type)) - pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type)); - - if (reindexing.clusters().get(cluster).ready().containsKey(type)) - setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type)); - } - if (clusters.containsKey(cluster)) - if (clusters.get(cluster).documentTypeStatus().containsKey(type)) - setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type)); - } - }); - } - - private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) { - object.setLong("readyMillis", readyStatus.ready().toEpochMilli()); - } - - private static void setStatus(Cursor object, ClusterReindexing.Status status) { - object.setLong("startedMillis", status.startedAt().toEpochMilli()); - status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli())); - status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state)); - status.message().ifPresent(message -> object.setString("message", message)); - status.progress().ifPresent(progress -> object.setDouble("progress", progress)); - } - - } - - private static JSONResponse createMessageResponse(String message) { - return new JSONResponse(Response.Status.OK) { { object.setString("message", message); } }; + private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) { + return Optional.ofNullable(request.getProperty("vespaVersion")) + .filter(s -> !s.isEmpty()) + .map(Version::fromString); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 1ea41b85983..0a69c7ed904 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; /** * Handler for getting tenant and application for a given hostname. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 1787431e841..76a39bbcadb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.http.HttpConfigRequest; import com.yahoo.vespa.config.server.http.HttpConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java index 71be471193a..3210f5eb8e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java @@ -15,6 +15,8 @@ import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.http.HttpConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java index bbc2efe9c36..425aa1c2bd2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java @@ -10,6 +10,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.http.HttpConfigRequest; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index b06d5c31ac6..20688f4f2f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -10,6 +10,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.http.v2.response.ListApplicationsResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 4e75234620f..f97e272b132 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -14,6 +14,7 @@ import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.TimeoutBudget; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java index 9f526bbdce8..4664fdf3557 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.http.v2.request.SessionContentRequestV2; /** * A handler that will return content or content status for files or directories diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 61f099fb8ea..ea7d11d9802 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.jdisc.HeaderFields; import com.yahoo.jdisc.application.UriPattern; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; @@ -17,13 +16,12 @@ import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; -import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.config.server.http.v2.response.SessionCreateResponse; import org.apache.hc.core5.http.ContentType; import java.net.URI; import java.time.Duration; import java.util.List; -import java.util.stream.Collectors; /** * A handler that is able to create a session from an application package, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index 6fa2075807f..0d6d5ed943a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index bb94f8d442a..a9961762f8b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -10,6 +10,10 @@ import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.v2.response.ListTenantsResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantCreateResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantDeleteResponse; +import com.yahoo.vespa.config.server.http.v2.response.TenantGetResponse; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.yolean.Exceptions; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java index af3ab0b1b83..b7ec6272c3a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; @@ -18,12 +18,12 @@ public class ApplicationContentRequest extends ContentRequest { private final ApplicationId applicationId; private final Zone zone; - ApplicationContentRequest(HttpRequest request, - long sessionId, - ApplicationId applicationId, - Zone zone, - String contentPath, - ApplicationFile applicationFile) { + public ApplicationContentRequest(HttpRequest request, + long sessionId, + ApplicationId applicationId, + Zone zone, + String contentPath, + ApplicationFile applicationFile) { super(request, sessionId, contentPath, applicationFile); this.applicationId = applicationId; this.zone = zone; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java index 8b482997044..d190ccaec12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import java.net.URI; @@ -45,7 +45,7 @@ public class HttpConfigRequests { } - static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) { + public static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) { Tenant tenant = tenantRepository.getTenant(request.getApplicationId().tenant()); if (tenant==null) throw new NotFoundException("No such tenant: "+request.getApplicationId().tenant()); return tenant.getRequestHandler(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java index 208b682df9a..cd9c44db85b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.collections.Tuple2; import com.yahoo.config.provision.ApplicationName; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java index c5a9d36f493..4f34b2fb332 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.TenantName; @@ -20,7 +20,7 @@ public class SessionContentRequestV2 extends ContentRequest { private final TenantName tenantName; private final long sessionId; - SessionContentRequestV2(HttpRequest request, + public SessionContentRequestV2(HttpRequest request, long sessionId, TenantName tenantName, String path, @@ -35,7 +35,7 @@ public class SessionContentRequestV2 extends ContentRequest { return "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; } - static String getContentPath(HttpRequest request) { + public static String getContentPath(HttpRequest request) { BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern); return bm.group(4); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java index 0fae091aa78..84071ca2a98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.request; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java new file mode 100644 index 00000000000..c2449d9a31d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.http.JSONResponse; + +public class ApplicationSuspendedResponse extends JSONResponse { + public ApplicationSuspendedResponse(boolean suspended) { + super(Response.Status.OK); + object.setBool("suspended", suspended); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java new file mode 100644 index 00000000000..375a6d5e57f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java @@ -0,0 +1,11 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.restapi.MessageResponse; + +public class DeleteApplicationResponse extends MessageResponse { + public DeleteApplicationResponse(ApplicationId applicationId) { + super("Application '" + applicationId + "' deleted"); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java index cdfdce91500..253e37f2b0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java new file mode 100644 index 00000000000..dc124891540 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.component.Version; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.server.http.JSONResponse; + +import java.util.List; +import java.util.Optional; + +public class GetApplicationResponse extends JSONResponse { + public GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) { + super(status); + object.setLong("generation", generation); + object.setString("applicationPackageFileReference", applicationPackageReference.orElse("")); + Cursor modelVersionArray = object.setArray("modelVersions"); + modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java index a4527305abb..6b1f7031d81 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java index 3789939429c..ec1b3c83604 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; +import java.util.Set; + /** * Tenant list response * @@ -13,7 +14,7 @@ import com.yahoo.slime.Cursor; */ public class ListTenantsResponse extends SlimeJsonResponse { - ListTenantsResponse(ImmutableSet<TenantName> tenants) { + public ListTenantsResponse(Set<TenantName> tenants) { Cursor tenantArray = slime.setObject().setArray("tenants"); tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java index 99b95f9244c..25e770c7211 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java @@ -1,5 +1,5 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java new file mode 100644 index 00000000000..e30f0f45098 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.http.JSONResponse; + +public class QuotaUsageResponse extends JSONResponse { + public QuotaUsageResponse(double usageRate) { + super(Response.Status.OK); + object.setDouble("rate", usageRate); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java new file mode 100644 index 00000000000..e73d4a9cb56 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2.response; + +import com.yahoo.jdisc.Response; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ClusterReindexing; +import com.yahoo.vespa.config.server.http.JSONResponse; + +import java.util.Map; +import java.util.Set; + +public class ReindexingResponse extends JSONResponse { + public ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing, + Map<String, ClusterReindexing> clusters) { + super(Response.Status.OK); + object.setBool("enabled", reindexing.enabled()); + Cursor clustersObject = object.setObject("clusters"); + documentTypes.forEach((cluster, types) -> { + Cursor clusterObject = clustersObject.setObject(cluster); + Cursor pendingObject = clusterObject.setObject("pending"); + Cursor readyObject = clusterObject.setObject("ready"); + + for (String type : types) { + Cursor statusObject = readyObject.setObject(type); + if (reindexing.clusters().containsKey(cluster)) { + if (reindexing.clusters().get(cluster).pending().containsKey(type)) + pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type)); + + if (reindexing.clusters().get(cluster).ready().containsKey(type)) + setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type)); + } + if (clusters.containsKey(cluster)) + if (clusters.get(cluster).documentTypeStatus().containsKey(type)) + setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type)); + } + }); + } + + private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) { + object.setLong("readyMillis", readyStatus.ready().toEpochMilli()); + } + + private static void setStatus(Cursor object, ClusterReindexing.Status status) { + object.setLong("startedMillis", status.startedAt().toEpochMilli()); + status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli())); + status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state)); + status.message().ifPresent(message -> object.setString("message", message)); + status.progress().ifPresent(progress -> object.setDouble("progress", progress)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java index 9c0fbdf2613..f14f9cc6575 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java index faf02a1ea4f..047962ea16b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java index 7bace4749a8..d219ff64b4e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -8,15 +8,16 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareAndActivateResponse extends SlimeJsonResponse { +public class SessionPrepareAndActivateResponse extends SlimeJsonResponse { - SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { + public SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { super(result.deployLogger().slime()); TenantName tenantName = applicationId.tenant(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java index a97cd37d3b4..5f50a5236fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -9,19 +9,20 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.Type; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareResponse extends SlimeJsonResponse { +public class SessionPrepareResponse extends SlimeJsonResponse { - SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { + public SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions()); } - SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { + public SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java index 6ff2b30075d..395a52df9f2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java index d21584c8cdc..fbfb4d581c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java index b918cab7828..3a50a68cb21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; +package com.yahoo.vespa.config.server.http.v2.response; import com.yahoo.config.provision.TenantName; import com.yahoo.restapi.MessageResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java index 7fc2c47c06c..2681952cae9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse; import java.net.URI; import java.util.Collection; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java index 5078fba8b38..4c111687c41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse; +import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse; import java.net.URI; import java.util.Collection; import java.util.function.Predicate; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 52fbb8e6781..a46b1cc9bdc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -90,7 +90,7 @@ public class SessionZooKeeperClient { return data.map(d -> Session.Status.parse(Utf8.toString(d))).orElse(Session.Status.NONE); } catch (Exception e) { log.log(Level.INFO, "Failed to read session status at " + sessionStatusPath.getAbsolute() + - ", will assume session has been removed: " + e.getMessage()); + ", will assume session has been removed: ", e); return Session.Status.NONE; } } diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 24acf79705d..f9eb5e3a0a5 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -175,6 +175,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.io.tmpdir=${VESPA_HOME}/tmp \ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 22a5b3795d9..46d94ec476b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -32,7 +32,7 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SecretStoreValidator; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.StaticResponse; -import com.yahoo.vespa.config.server.http.v2.ApplicationHandler.ReindexingResponse; +import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -60,7 +60,6 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Stream; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.container.jdisc.HttpRequest.createTestRequest; import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -72,9 +71,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author hmusum @@ -343,10 +341,9 @@ public class ApplicationHandlerTest { } @Test - public void testClusterControllerStatus() throws Exception { + public void testServiceStatus() throws Exception { applicationRepository.deploy(testApp, prepareParams(applicationId)); String host = "foo.yahoo.com"; - String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1"; HttpProxy mockHttpProxy = mock(HttpProxy.class); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) @@ -356,11 +353,19 @@ public class ApplicationHandlerTest { .withHttpProxy(mockHttpProxy) .build(); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); - when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName), eq("clustercontroller-status/v1/clusterName1"))) - .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); + doAnswer(invoc -> new StaticResponse(200, "text/html", "<html>" + + "host=" + invoc.getArgument(1, String.class) + "," + + "service=" + invoc.getArgument(2, String.class) + "," + + "path=" + invoc.getArgument(3, String.class) + "</html>")).when(mockHttpProxy).get(any(), any(), any(), any()); - HttpResponse response = mockHandler.handle(createTestRequest(url, GET)); - assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>"); + HttpResponse response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/container-clustercontroller/" + host + "/status/some/path/clusterName1", GET)); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=clustercontroller-status/v1/some/path/clusterName1</html>"); + + response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/status/something", GET)); + assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=something</html>"); + + response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/fake-service/" + host + "/status/something", GET)); + assertHttpStatusCodeAndMessage(response, 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No status page for service: fake-service\"}"); } @Test diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java new file mode 100644 index 00000000000..28199fd727b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.entity; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-core/pom.xml b/container-core/pom.xml index 2b87d79daa4..c7fe2998530 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -140,10 +140,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> @@ -211,6 +207,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -262,11 +264,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpmime</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> <scope>test</scope> @@ -302,17 +299,6 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents.client5</groupId> - <artifactId>httpclient5</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 9f6f5cf7ea1..08a468a3031 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -117,8 +117,11 @@ public class HandlersConfigurerDi { @Override public void installPlatformBundles(Collection<String> bundlePaths) { - log.fine("Installing platform bundles."); - platformBundleLoader.useBundles(new ArrayList<>(bundlePaths)); + // Don't install physical bundles for test frameworks, where all platform bundles are on the classpath. + if (osgiFramework.isFelixFramework()) { + log.fine("Installing platform bundles."); + platformBundleLoader.useBundles(new ArrayList<>(bundlePaths)); + } } @Override diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java new file mode 100644 index 00000000000..9610648ad41 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.AbstractRequestHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; + +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author bjorncs + */ +class EchoRequestHandler extends AbstractRequestHandler { + @Override + public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { + int port = request.getUri().getPort(); + Response response = new Response(OK); + response.headers().put("Jdisc-Local-Port", Integer.toString(port)); + return handler.handleResponse(response); + } +} + diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 0f625b5c3df..bd3000a0188 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -5,14 +5,12 @@ import com.google.inject.AbstractModule; import com.google.inject.Module; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogEntry; -import com.yahoo.container.logging.ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry; import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; import com.yahoo.jdisc.References; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingSetSelector; -import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; @@ -28,12 +26,7 @@ import com.yahoo.jdisc.http.HttpResponse; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.server.jetty.JettyTestDriver.TlsClientAuth; import com.yahoo.jdisc.service.BindingSetNotFoundException; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.Pkcs10Csr; -import com.yahoo.security.Pkcs10CsrBuilder; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; @@ -42,39 +35,22 @@ import org.apache.hc.client5.http.entity.mime.FormBodyPartBuilder; import org.apache.hc.client5.http.entity.mime.StringBody; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.H2AsyncClientBuilder; -import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; -import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; -import org.apache.hc.core5.http2.HttpVersionPolicy; import org.assertj.core.api.Assertions; -import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1; -import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; -import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.server.handler.AbstractHandlerContainer; -import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLHandshakeException; -import javax.security.auth.x500.X500Principal; import java.io.IOException; -import java.math.BigInteger; import java.net.BindException; import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.security.KeyPair; -import java.security.PrivateKey; -import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -85,10 +61,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Pattern; import static com.yahoo.jdisc.Response.Status.GATEWAY_TIMEOUT; @@ -105,8 +78,8 @@ import static com.yahoo.jdisc.http.HttpHeaders.Names.X_DISABLE_CHUNKING; import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED; import static com.yahoo.jdisc.http.HttpHeaders.Values.CLOSE; import static com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.ResponseValidator; -import static com.yahoo.security.KeyAlgorithm.EC; -import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; import static org.cthul.matchers.CthulMatchers.containsPattern; import static org.cthul.matchers.CthulMatchers.matchesPattern; import static org.hamcrest.CoreMatchers.containsString; @@ -114,13 +87,10 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.anyOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -133,8 +103,6 @@ import static org.mockito.Mockito.when; */ public class HttpServerTest { - private static final Logger log = Logger.getLogger(HttpServerTest.class.getName()); - @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -666,205 +634,6 @@ public class HttpServerTest { } @Test - public void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .build(); - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1"); - assumeTrue("TLSv1.1 must be enabled in installed JDK", tlsv11Enabled); - - assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(certificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake() throws IOException { - Path serverPrivateKeyFile = tmpFolder.newFile().toPath(); - Path serverCertificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(serverPrivateKeyFile, serverCertificateFile); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(serverCertificateFile, serverPrivateKeyFile, metricConsumer, connectionLog); - - Path clientPrivateKeyFile = tmpFolder.newFile().toPath(); - Path clientCertificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile); - - SSLContext clientCtx = new SslContextBuilder() - .withKeyStore(clientPrivateKeyFile, clientCertificateFile) - .withTrustStore(serverCertificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - assertSslHandshakeFailurePresent( - connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType()); - } - - @Test - public void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake() throws IOException { - Path rootPrivateKeyFile = tmpFolder.newFile().toPath(); - Path rootCertificateFile = tmpFolder.newFile().toPath(); - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS); - generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile, privateKeyFile, certificateFile, notAfter); - var metricConsumer = new MetricConsumerMock(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslTestDriver(rootCertificateFile, rootPrivateKeyFile, metricConsumer, connectionLog); - - SSLContext clientCtx = new SslContextBuilder() - .withTrustStore(rootCertificateFile) - .withKeyStore(privateKeyFile, certificateFile) - .build(); - - assertHttpsRequestTriggersSslHandshakeException( - driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); - verify(metricConsumer.mockitoMock(), atLeast(1)) - .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); - assertTrue(driver.close()); - Assertions.assertThat(connectionLog.logEntries()).hasSize(1); - - } - - @Test - public void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false); - - String proxiedRemoteAddress = "192.168.0.100"; - int proxiedRemotePort = 12345; - sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, proxiedRemotePort)); - sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, proxiedRemotePort)); - assertTrue(driver.close()); - - assertEquals(2, requestLogMock.entries().size()); - assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort); - assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort); - Assertions.assertThat(connectionLog.logEntries()).hasSize(2); - assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort); - assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get()); - assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort); - assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get()); - } - - @Test - public void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true); - - String proxiedRemoteAddress = "192.168.0.100"; - sendJettyClientRequest(driver, certificateFile, null); - sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, 12345)); - sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, 12345)); - assertTrue(driver.close()); - - assertEquals(3, requestLogMock.entries().size()); - assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0); - assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0); - assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0); - Assertions.assertThat(connectionLog.logEntries()).hasSize(3); - assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0); - assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345); - assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345); - } - - @Test - public void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception { - Path privateKeyFile = tmpFolder.newFile().toPath(); - Path certificateFile = tmpFolder.newFile().toPath(); - generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); - InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); - InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); - JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, connectionLog, /*mixedMode*/false); - - String proxiedRemoteAddress = "192.168.0.100"; - int proxiedRemotePort = 12345; - String proxyLocalAddress = "10.0.0.10"; - int proxyLocalPort = 23456; - V2.Tag v2Tag = new V2.Tag(V2.Tag.Command.PROXY, null, V2.Tag.Protocol.STREAM, - proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null); - ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag); - assertTrue(driver.close()); - - int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port")); - assertNotEquals(proxyLocalPort, clientPort); - assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue()); - } - - @Test public void requireThatConnectionIsTrackedInConnectionLog() throws Exception { Path privateKeyFile = tmpFolder.newFile().toPath(); Path certificateFile = tmpFolder.newFile().toPath(); @@ -932,41 +701,6 @@ public class HttpServerTest { .set(MetricDefinitions.REQUESTS_PER_CONNECTION, 1L, MetricConsumerMock.STATIC_CONTEXT); } - private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag) - throws Exception { - HttpClient client = createJettyHttpClient(certificateFile); - try { - int maxAttempts = 3; - for (int attempt = 0; attempt < maxAttempts; attempt++) { - try { - ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) - .tag(tag) - .send(); - assertEquals(200, response.getStatus()); - return response; - } catch (ExecutionException e) { - // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. - // We have been unable to reproduce this locally. The cause is therefor currently unknown. - log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); - Thread.sleep(10); - } - } - throw new AssertionError("Failed to send request, see log for details"); - } finally { - client.stop(); - } - } - - // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. - private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception { - SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client(); - clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); - clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); - - HttpClient client = new HttpClient(clientSslCtxFactory); - client.start(); - return client; - } private static CloseableHttpAsyncClient createHttp2Client(JettyTestDriver driver) { TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() @@ -980,58 +714,6 @@ public class HttpServerTest { return client; } - private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) { - assertEquals(expectedAddress, entry.peerAddress().get()); - if (expectedPort > 0) { - assertEquals(expectedPort, entry.peerPort().getAsInt()); - } - } - - private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) { - if (expectedAddress != null) { - Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress); - } else { - Assertions.assertThat(entry.remoteAddress()).isEmpty(); - } - if (expectedPort > 0) { - Assertions.assertThat(entry.remotePort()).hasValue(expectedPort); - } else { - Assertions.assertThat(entry.remotePort()).isEmpty(); - } - } - - private static void assertSslHandshakeFailurePresent( - ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) { - Assertions.assertThat(entry.sslHandshakeFailure()).isPresent(); - ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get(); - assertEquals(expectedType, failure.type()); - ExceptionEntry exceptionEntry = failure.exceptionChain().get(0); - assertEquals(expectedException.getName(), exceptionEntry.name()); - } - - private static JettyTestDriver createSslWithProxyProtocolTestDriver( - Path certificateFile, Path privateKeyFile, RequestLog requestLog, - ConnectionLog connectionLog, boolean mixedMode) { - ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() - .http2Enabled(true) - .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder() - .enabled(true) - .mixedMode(mixedMode)) - .ssl(new ConnectorConfig.Ssl.Builder() - .enabled(true) - .privateKeyFile(privateKeyFile.toString()) - .certificateFile(certificateFile.toString()) - .caCertificateFile(certificateFile.toString())); - return JettyTestDriver.newConfiguredInstance( - new EchoRequestHandler(), - new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)), - connectorConfig, - binder -> { - binder.bind(RequestLog.class).toInstance(requestLog); - binder.bind(ConnectionLog.class).toInstance(connectionLog); - }); - } - private static JettyTestDriver createSslWithTlsClientAuthenticationEnforcer(Path certificateFile, Path privateKeyFile) { ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() .tlsClientAuthEnforcer( @@ -1051,63 +733,6 @@ public class HttpServerTest { binder -> {}); } - private static JettyTestDriver createSslTestDriver( - Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) throws IOException { - Module extraModule = binder -> { - binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock()); - binder.bind(ConnectionLog.class).toInstance(connectionLog); - }; - return JettyTestDriver.newInstanceWithSsl( - new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, TlsClientAuth.NEED, extraModule); - } - - private static void assertHttpsRequestTriggersSslHandshakeException( - JettyTestDriver testDriver, - SSLContext sslContext, - String protocolOverride, - String cipherOverride, - String expectedExceptionSubstring) throws IOException { - List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null; - List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null; - try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) { - client.get("/status.html"); - fail("SSLHandshakeException expected"); - } catch (SSLHandshakeException e) { - assertThat(e.getMessage(), containsString(expectedExceptionSubstring)); - } catch (SSLException e) { - // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread. - log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e); - // Only ignore a subset of exceptions - assertThat(e.getMessage(), anyOf(containsString("readHandshakeRecord"), containsString("Broken pipe"))); - } - } - - private static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException { - KeyPair keyPair = KeyUtils.generateKeypair(EC); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); - - X509Certificate certificate = X509CertificateBuilder - .fromKeypair( - keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) - .build(); - Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); - } - - private static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile, - Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException { - generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile); - X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile)); - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile)); - - KeyPair keyPair = KeyUtils.generateKeypair(EC); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); - Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build(); - X509Certificate certificate = X509CertificateBuilder - .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE) - .build(); - Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); - } - private static RequestHandler mockRequestHandler() { final RequestHandler mockRequestHandler = mock(RequestHandler.class); when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE); @@ -1237,17 +862,6 @@ public class HttpServerTest { } } - private static class EchoRequestHandler extends AbstractRequestHandler { - - @Override - public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { - int port = request.getUri().getPort(); - Response response = new Response(OK); - response.headers().put("Jdisc-Local-Port", Integer.toString(port)); - return handler.handleResponse(response); - } - } - private static class OkRequestHandler extends AbstractRequestHandler { @Override public ContentChannel handleRequest(Request request, ResponseHandler handler) { diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java new file mode 100644 index 00000000000..d29abea024e --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java @@ -0,0 +1,198 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.container.logging.ConnectionLogEntry; +import com.yahoo.container.logging.RequestLog; +import com.yahoo.container.logging.RequestLogEntry; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.security.SslContextBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.assertj.core.api.Assertions; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class ProxyProtocolTest { + + private static final Logger log = Logger.getLogger(ProxyProtocolTest.class.getName()); + + private static Path privateKeyFile; + private static Path certificateFile; + private InMemoryConnectionLog connectionLog; + private InMemoryRequestLog requestLogMock; + private JettyTestDriver driver; + + @BeforeAll + static void generateCrypto(@TempDir Path tmpFolder) throws IOException { + privateKeyFile = tmpFolder.resolve("key.pem"); + certificateFile = tmpFolder.resolve("cert.pem"); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + } + + @BeforeEach + void initializeServer() { + requestLogMock = new InMemoryRequestLog(); + connectionLog = new InMemoryConnectionLog(); + driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false); + } + + @Test + void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception { + String proxiedRemoteAddress = "192.168.0.100"; + int proxiedRemotePort = 12345; + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, proxiedRemotePort)); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, proxiedRemotePort)); + assertTrue(driver.close()); + + assertEquals(2, requestLogMock.entries().size()); + assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort); + assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort); + Assertions.assertThat(connectionLog.logEntries()).hasSize(2); + assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort); + assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get()); + assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort); + assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get()); + } + + @Test + void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception { + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + InMemoryRequestLog requestLogMock = new InMemoryRequestLog(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true); + + String proxiedRemoteAddress = "192.168.0.100"; + sendJettyClientRequest(driver, certificateFile, null); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, 12345)); + sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, 12345)); + assertTrue(driver.close()); + + assertEquals(3, requestLogMock.entries().size()); + assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0); + assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0); + assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0); + Assertions.assertThat(connectionLog.logEntries()).hasSize(3); + assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0); + assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345); + assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345); + } + + @Test + void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception { + String proxiedRemoteAddress = "192.168.0.100"; + int proxiedRemotePort = 12345; + String proxyLocalAddress = "10.0.0.10"; + int proxyLocalPort = 23456; + ProxyProtocolClientConnectionFactory.V2.Tag v2Tag = new ProxyProtocolClientConnectionFactory.V2.Tag(ProxyProtocolClientConnectionFactory.V2.Tag.Command.PROXY, null, ProxyProtocolClientConnectionFactory.V2.Tag.Protocol.STREAM, + proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null); + ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag); + assertTrue(driver.close()); + + int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port")); + assertNotEquals(proxyLocalPort, clientPort); + assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue()); + } + + private static JettyTestDriver createSslWithProxyProtocolTestDriver( + Path certificateFile, Path privateKeyFile, RequestLog requestLog, + ConnectionLog connectionLog, boolean mixedMode) { + ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder() + .http2Enabled(true) + .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder() + .enabled(true) + .mixedMode(mixedMode)) + .ssl(new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyFile.toString()) + .certificateFile(certificateFile.toString()) + .caCertificateFile(certificateFile.toString())); + return JettyTestDriver.newConfiguredInstance( + new EchoRequestHandler(), + new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)), + connectorConfig, + binder -> { + binder.bind(RequestLog.class).toInstance(requestLog); + binder.bind(ConnectionLog.class).toInstance(connectionLog); + }); + } + + private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag) + throws Exception { + HttpClient client = createJettyHttpClient(certificateFile); + try { + int maxAttempts = 3; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) + .tag(tag) + .send(); + assertEquals(200, response.getStatus()); + return response; + } catch (ExecutionException e) { + // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. + // We have been unable to reproduce this locally. The cause is therefor currently unknown. + log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); + Thread.sleep(10); + } + } + throw new AssertionError("Failed to send request, see log for details"); + } finally { + client.stop(); + } + } + + // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. + private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception { + SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client(); + clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build()); + + HttpClient client = new HttpClient(clientSslCtxFactory); + client.start(); + return client; + } + + private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) { + assertEquals(expectedAddress, entry.peerAddress().get()); + if (expectedPort > 0) { + assertEquals(expectedPort, entry.peerPort().getAsInt()); + } + } + + private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) { + if (expectedAddress != null) { + Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress); + } else { + Assertions.assertThat(entry.remoteAddress()).isEmpty(); + } + if (expectedPort > 0) { + Assertions.assertThat(entry.remotePort()).hasValue(expectedPort); + } else { + Assertions.assertThat(entry.remotePort()).isEmpty(); + } + } + + +} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java new file mode 100644 index 00000000000..0f71dd87d00 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java @@ -0,0 +1,189 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.yahoo.container.logging.ConnectionLogEntry; +import com.yahoo.security.SslContextBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; +import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +class SslHandshakeMetricsTest { + + private static final Logger log = Logger.getLogger(SslHandshakeMetricsTest.class.getName()); + private static Path privateKeyFile; + private static Path certificateFile; + + @BeforeAll + static void generateCrypto(@TempDir Path tmpFolder) throws IOException { + privateKeyFile = tmpFolder.resolve("private-key.pem"); + certificateFile = tmpFolder.resolve("certificate.pem"); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + } + + @Test + void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .build(); + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: bad_certificate"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(privateKeyFile, certificateFile) + .build(); + + boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1"); + assumeTrue(tlsv11Enabled, "TLSv1.1 must be enabled in installed JDK"); + + assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(privateKeyFile, certificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake(@TempDir Path tmpFolder) throws IOException { + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + Path clientPrivateKeyFile = tmpFolder.resolve("client-key.pem"); + Path clientCertificateFile = tmpFolder.resolve("client-cert.pem"); + generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile); + + SSLContext clientCtx = new SslContextBuilder() + .withKeyStore(clientPrivateKeyFile, clientCertificateFile) + .withTrustStore(certificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + assertSslHandshakeFailurePresent( + connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType()); + } + + @Test + void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake(@TempDir Path tmpFolder) throws IOException { + Path endEntityKeyFile = tmpFolder.resolve("client-key.pem"); + Path endEntitycertificateFile = tmpFolder.resolve("client-cert.pem"); + Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile, endEntityKeyFile, endEntitycertificateFile, notAfter); + var metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + + SSLContext clientCtx = new SslContextBuilder() + .withTrustStore(certificateFile) + .withKeyStore(endEntityKeyFile, endEntitycertificateFile) + .build(); + + assertHttpsRequestTriggersSslHandshakeException( + driver, clientCtx, null, null, "Received fatal alert: certificate_unknown"); + verify(metricConsumer.mockitoMock(), atLeast(1)) + .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT); + assertTrue(driver.close()); + assertThat(connectionLog.logEntries()).hasSize(1); + } + + + private static void assertHttpsRequestTriggersSslHandshakeException( + JettyTestDriver testDriver, + SSLContext sslContext, + String protocolOverride, + String cipherOverride, + String expectedExceptionSubstring) throws IOException { + List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null; + List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null; + try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) { + client.get("/status.html"); + fail("SSLHandshakeException expected"); + } catch (SSLHandshakeException e) { + assertThat(e.getMessage()).contains(expectedExceptionSubstring); + } catch (SSLException e) { + // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread. + log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e); + // Only ignore a subset of exceptions + assertTrue(e.getMessage().contains("readHandshakeRecord") || e.getMessage().contains("Broken pipe"), e.getMessage()); + } + } + + private static void assertSslHandshakeFailurePresent( + ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) { + assertThat(entry.sslHandshakeFailure()).isPresent(); + ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get(); + assertEquals(expectedType, failure.type()); + ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry exceptionEntry = failure.exceptionChain().get(0); + assertEquals(expectedException.getName(), exceptionEntry.name()); + } + +} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java new file mode 100644 index 00000000000..626ab521773 --- /dev/null +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.server.jetty; + +import com.google.inject.Module; +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.Pkcs10Csr; +import com.yahoo.security.Pkcs10CsrBuilder; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; + +import javax.security.auth.x500.X500Principal; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static com.yahoo.security.KeyAlgorithm.EC; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; + +/** + * @author bjorncs + */ +class Utils { + + private Utils() {} + + static JettyTestDriver createSslTestDriver( + Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) { + Module extraModule = binder -> { + binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock()); + binder.bind(ConnectionLog.class).toInstance(connectionLog); + }; + return JettyTestDriver.newInstanceWithSsl( + new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, JettyTestDriver.TlsClientAuth.NEED, extraModule); + } + + static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException { + KeyPair keyPair = KeyUtils.generateKeypair(EC); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + + X509Certificate certificate = X509CertificateBuilder + .fromKeypair( + keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + } + + static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile, + Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException { + X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile)); + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile)); + + KeyPair keyPair = KeyUtils.generateKeypair(EC); + Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build(); + X509Certificate certificate = X509CertificateBuilder + .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate)); + } +} diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 5446c9e1698..b255b6af02a 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -170,14 +170,11 @@ configgen.jar, config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, - container-jersey2-jar-with-dependencies.jar, container-search-and-docproc-jar-with-dependencies.jar, container-search-gui-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, - model-evaluation-jar-with-dependencies.jar, - model-integration-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, security-utils-jar-with-dependencies.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index f43c1fcffd6..0e4c4446778 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -18,6 +18,7 @@ import com.yahoo.container.di.config.Subscriber; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.container.jdisc.component.Deconstructor; +import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.container.jdisc.metric.DisableGuiceMetric; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.application.Application; @@ -76,6 +77,7 @@ public final class ConfiguredApplication implements Application { // to config to make sure that container will be registered in slobrok (by {@link com.yahoo.jrt.slobrok.api.Register}) // if slobrok config changes (typically slobroks moving to other nodes) private final Optional<SlobrokConfigSubscriber> slobrokConfigSubscriber; + private final SessionCache sessionCache; //TODO: FilterChainRepository should instead always be set up in the model. private final FilterChainRepository defaultFilterChainRepository = @@ -125,6 +127,7 @@ public final class ConfiguredApplication implements Application { this.slobrokConfigSubscriber = (subscriberFactory instanceof CloudSubscriberFactory) ? Optional.of(new SlobrokConfigSubscriber(configId)) : Optional.empty(); + this.sessionCache = new SessionCache(configId); this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext())); } @@ -346,6 +349,7 @@ public final class ConfiguredApplication implements Application { bind(OsgiFramework.class).toInstance(restrictedOsgiFramework); bind(com.yahoo.jdisc.Timer.class).toInstance(timerSingleton); bind(FilterChainRepository.class).toInstance(defaultFilterChainRepository); + bind(SessionCache.class).toInstance(sessionCache); // Needed by e.g. FeedHandler } }); } diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 4518868cb81..223124c29d6 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -215,6 +215,7 @@ exec $numactlcmd $envcmd java \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.io.tmpdir="${VESPA_HOME}/tmp" \ diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index ab3d09af178..d65b2f7cc12 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -1,24 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.messagebus; -import com.google.inject.Inject; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUtil; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.References; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.SharedResource; +import java.util.logging.Level; +import com.yahoo.messagebus.ConfigAgent; import com.yahoo.messagebus.DynamicThrottlePolicy; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.MessageBusParams; -import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.StaticThrottlePolicy; @@ -28,12 +26,9 @@ import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.shared.SharedIntermediateSession; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import java.util.HashMap; import java.util.Map; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -42,19 +37,24 @@ import java.util.logging.Logger; * @author Steinar Knutsen * @author Einar Rosenvinge */ -// TODO jonmv: Remove this: only used with more than one entry by FeedHandlerV3, where only timeout varies. -// rant: This whole construct is because DI at one point didn't exist, so getting hold of a shared resource -// or session was hard(?), and one resorted to routing through the Container, using URIs, to the correct -// MbusClient, with or without throttling. This introduced the problem of ownership during shutdown, -// which was solved with manual reference counting. This is all much better solved with DI, which (now) -// owns everything, and does component shutdown in reverse construction order, which is always right. -// So for the sake everyone's mental health, this should all just be removed now! I suspect this is -// even the case for Request; we can track in handlers, and warn when requests have been misplaced. +// TODO jonmv: Remove this? Only used sensibly by FeedHandlerV3, where only timeout varies. public final class SessionCache extends AbstractComponent { private static final Logger log = Logger.getLogger(SessionCache.class.getName()); - private final SharedMessageBus messageBus; + //config + private final String messagebusConfigId; + private final String slobrokConfigId; + private final String identity; + private final String containerMbusConfigId; + private final String documentManagerConfigId; + private final String loadTypeConfigId; + private final DocumentTypeManager documentTypeManager; + + // initialized in start() + private ConfigAgent configAgent; + private SharedMessageBus messageBus; + private final Object intermediateLock = new Object(); private final Map<String, SharedIntermediateSession> intermediates = new HashMap<>(); private final IntermediateSessionCreator intermediatesCreator = new IntermediateSessionCreator(); @@ -63,48 +63,52 @@ public final class SessionCache extends AbstractComponent { private final Map<SourceSessionKey, SharedSourceSession> sources = new HashMap<>(); private final SourceSessionCreator sourcesCreator = new SourceSessionCreator(); - @Inject - public SessionCache(ContainerMbusConfig containerMbusConfig, DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig, - MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, - DistributionConfig distributionConfig) { - this(containerMbusConfig, documentmanagerConfig, loadTypeConfig, slobroksConfig, - messagebusConfig, policiesConfig, distributionConfig, System.getProperty("config.id")); //: + public SessionCache(String messagebusConfigId, String slobrokConfigId, String identity, + String containerMbusConfigId, String documentManagerConfigId, + String loadTypeConfigId, + DocumentTypeManager documentTypeManager) { + this.messagebusConfigId = messagebusConfigId; + this.slobrokConfigId = slobrokConfigId; + this.identity = identity; + this.containerMbusConfigId = containerMbusConfigId; + this.documentManagerConfigId = documentManagerConfigId; + this.loadTypeConfigId = loadTypeConfigId; + this.documentTypeManager = documentTypeManager; } - public SessionCache(ContainerMbusConfig containerMbusConfig, DocumentmanagerConfig documentmanagerConfig, - LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig, - MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, - DistributionConfig distributionConfig, String identity) { - this.messageBus = createSharedMessageBus(containerMbusConfig, - messagebusConfig, - slobroksConfig, - identity, - new DocumentProtocol(new DocumentTypeManager(documentmanagerConfig), - new LoadTypeSet(loadTypeConfig), - policiesConfig, - distributionConfig)); + public SessionCache(final String identity) { + this(identity, identity, identity, identity, identity, identity, new DocumentTypeManager()); } - public SessionCache(ContainerMbusConfig containerMbusConfig, SlobroksConfig slobroksConfig, - MessagebusConfig messagebusConfig, String identity, Protocol protocol) { - this.messageBus = createSharedMessageBus(containerMbusConfig, - messagebusConfig, - slobroksConfig, - identity, - protocol); + public void deconstruct() { + if (configAgent != null) { + configAgent.shutdown(); + } } - public void deconstruct() { - messageBus.release(); + + private void start() { + ContainerMbusConfig mbusConfig = ConfigGetter.getConfig(ContainerMbusConfig.class, containerMbusConfigId); + if (documentManagerConfigId != null) { + documentTypeManager.configure(documentManagerConfigId); + } + LoadTypeSet loadTypeSet = new LoadTypeSet(loadTypeConfigId); + DocumentProtocol protocol = new DocumentProtocol(documentTypeManager, identity, loadTypeSet); + messageBus = createSharedMessageBus(mbusConfig, slobrokConfigId, identity, protocol); + // TODO: stop doing subscriptions to config when that is to be solved in slobrok as well + configAgent = new ConfigAgent(messagebusConfigId, messageBus.messageBus()); + configAgent.subscribe(); + } + + + private boolean isStarted() { + return messageBus != null; } private static SharedMessageBus createSharedMessageBus(ContainerMbusConfig mbusConfig, - MessagebusConfig messagebusConfig, - SlobroksConfig slobroksConfig, String identity, + String slobrokConfigId, String identity, Protocol protocol) { - MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol) - .setMessageBusConfig(messagebusConfig); + MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol); int maxPendingSize = DocumentUtil .calculateMaxPendingSize(mbusConfig.maxConcurrentFactor(), mbusConfig.documentExpansionFactor(), @@ -115,7 +119,7 @@ public final class SessionCache extends AbstractComponent { mbusParams.setMaxPendingSize(maxPendingSize); RPCNetworkParams netParams = new RPCNetworkParams() - .setSlobroksConfig(slobroksConfig) + .setSlobrokConfigId(slobrokConfigId) .setIdentity(new Identity(identity)) .setListenPort(mbusConfig.port()) .setNumTargetsPerSpec(mbusConfig.numconnectionspertarget()) @@ -140,10 +144,20 @@ public final class SessionCache extends AbstractComponent { } ReferencedResource<SharedIntermediateSession> retainIntermediate(final IntermediateSessionParams p) { + synchronized (this) { + if (!isStarted()) { + start(); + } + } return intermediatesCreator.retain(intermediateLock, intermediates, p); } public ReferencedResource<SharedSourceSession> retainSource(final SourceSessionParams p) { + synchronized (this) { + if (!isStarted()) { + start(); + } + } return sourcesCreator.retain(sourceLock, sources, p); } diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java index ea0ed7eadc8..111805d61b0 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java @@ -33,7 +33,7 @@ public class ClientTestDriver { this.server = server; MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol); - RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(server.slobroksConfig()); + RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(server.slobrokId()); SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams); session = mbus.newSourceSession(new SourceSessionParams()); client = new MbusClient(session); @@ -128,4 +128,7 @@ public class ClientTestDriver { return new ClientTestDriver(RemoteServer.newInstanceWithInternSlobrok(), protocol); } + public static ClientTestDriver newInstanceWithExternSlobrok(String slobrokId) { + return new ClientTestDriver(RemoteServer.newInstanceWithExternSlobrok(slobrokId), new SimpleProtocol()); + } } diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java index 6cd8fb8f34d..57d0abd980b 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java @@ -1,17 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.jdisc.test; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.MessageBus; -import com.yahoo.messagebus.MessageBusParams; -import com.yahoo.messagebus.Protocol; -import com.yahoo.messagebus.Reply; -import com.yahoo.messagebus.Result; -import com.yahoo.messagebus.SourceSession; -import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.*; import com.yahoo.messagebus.network.local.LocalNetwork; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; @@ -25,14 +17,16 @@ import java.util.concurrent.TimeUnit; public class RemoteClient { private final Slobrok slobrok; + private final String slobrokId; private final MessageBus mbus; private final ReplyQueue queue = new ReplyQueue(); private final SourceSession session; - private RemoteClient(Protocol protocol, boolean network) { - this.slobrok = newSlobrok(); + private RemoteClient(Slobrok slobrok, String slobrokId, Protocol protocol, boolean network) { + this.slobrok = slobrok; + this.slobrokId = slobrok != null ? slobrok.configId() : slobrokId; mbus = network - ? new MessageBus(new RPCNetwork(new RPCNetworkParams().setSlobroksConfig(slobroksConfig())), + ? new MessageBus(new RPCNetwork(new RPCNetworkParams().setSlobrokConfigId(this.slobrokId)), new MessageBusParams().addProtocol(protocol)) : new MessageBus(new LocalNetwork(), new MessageBusParams().addProtocol(protocol)); session = mbus.createSourceSession(new SourceSessionParams().setThrottlePolicy(null).setReplyHandler(queue)); @@ -46,22 +40,28 @@ public class RemoteClient { return queue.awaitReply(timeout, unit); } - public SlobroksConfig slobroksConfig() { - return TestUtils.configFor(slobrok); + public String slobrokId() { + return slobrokId; } public void close() { session.destroy(); mbus.destroy(); - slobrok.stop(); + if (slobrok != null) { + slobrok.stop(); + } } public static RemoteClient newInstanceWithInternSlobrok(boolean network) { - return new RemoteClient(new SimpleProtocol(), network); + return new RemoteClient(newSlobrok(), null, new SimpleProtocol(), network); + } + + public static RemoteClient newInstanceWithExternSlobrok(String slobrokId, boolean network) { + return new RemoteClient(null, slobrokId, new SimpleProtocol(), network); } public static RemoteClient newInstanceWithProtocolAndInternSlobrok(Protocol protocol, boolean network) { - return new RemoteClient(protocol, network); + return new RemoteClient(newSlobrok(), null, protocol, network); } private static Slobrok newSlobrok() { diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java index 0fb23e33709..1f0f82c4903 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.jdisc.test; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.DestinationSession; -import com.yahoo.messagebus.DestinationSessionParams; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.MessageBus; -import com.yahoo.messagebus.MessageBusParams; -import com.yahoo.messagebus.Protocol; -import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.*; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; @@ -24,14 +17,16 @@ import java.util.concurrent.TimeUnit; public class RemoteServer { private final Slobrok slobrok; + private final String slobrokId; private final MessageBus mbus; private final MessageQueue queue = new MessageQueue(); private final DestinationSession session; - private RemoteServer(Protocol protocol, String identity) { - this.slobrok = newSlobrok(); + private RemoteServer(Slobrok slobrok, String slobrokId, Protocol protocol, String identity) { + this.slobrok = slobrok; + this.slobrokId = slobrok != null ? slobrok.configId() : slobrokId; mbus = new MessageBus(new RPCNetwork(new RPCNetworkParams() - .setSlobroksConfig(slobroksConfig()) + .setSlobrokConfigId(this.slobrokId) .setIdentity(new Identity(identity))), new MessageBusParams().addProtocol(protocol)); session = mbus.createDestinationSession(new DestinationSessionParams().setMessageHandler(queue)); @@ -53,22 +48,32 @@ public class RemoteServer { session.reply(reply); } - public SlobroksConfig slobroksConfig() { - return TestUtils.configFor(slobrok); + public String slobrokId() { + return slobrokId; } public void close() { session.destroy(); mbus.destroy(); - slobrok.stop(); + if (slobrok != null) { + slobrok.stop(); + } } public static RemoteServer newInstanceWithInternSlobrok() { - return new RemoteServer(new SimpleProtocol(), "remote"); + return new RemoteServer(newSlobrok(), null, new SimpleProtocol(), "remote"); + } + + public static RemoteServer newInstanceWithExternSlobrok(String slobrokId) { + return new RemoteServer(null, slobrokId, new SimpleProtocol(), "remote"); + } + + public static RemoteServer newInstance(String slobrokId, String identity, Protocol protocol) { + return new RemoteServer(null, slobrokId, protocol, identity); } - public static RemoteServer newInstance(String identity, Protocol protocol) { - return new RemoteServer(protocol, identity); + public static RemoteServer newInstanceWithProtocol(Protocol protocol) { + return new RemoteServer(newSlobrok(), null, protocol, "remote"); } private static Slobrok newSlobrok() { diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java index fa0dd37ed13..e59db28e886 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java @@ -42,7 +42,7 @@ public class ServerTestDriver { } MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol); - RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(client.slobroksConfig()); + RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(client.slobrokId()); SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams); ServerSession session = mbus.newDestinationSession(new DestinationSessionParams()); server = new MbusServer(driver, session); @@ -130,6 +130,13 @@ public class ServerTestDriver { guiceModules); } + public static ServerTestDriver newInstanceWithExternSlobrok(String slobrokId, RequestHandler requestHandler, + boolean network, Module... guiceModules) + { + return new ServerTestDriver(RemoteClient.newInstanceWithExternSlobrok(slobrokId, network), + true, requestHandler, new SimpleProtocol(), guiceModules); + } + public static ServerTestDriver newInactiveInstance(boolean network, Module... guiceModules) { return new ServerTestDriver(RemoteClient.newInstanceWithInternSlobrok(network), false, null, new SimpleProtocol(), guiceModules); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java deleted file mode 100644 index 85ad241259f..00000000000 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.messagebus.jdisc.test; - -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.slobrok.server.Slobrok; - -/** - * @author jonmv - */ -public class TestUtils { - - private TestUtils() { } - - public static SlobroksConfig configFor(Slobrok slobrok) { - return new SlobroksConfig.Builder().slobrok(new SlobroksConfig.Slobrok.Builder() - .connectionspec(new Spec("localhost", slobrok.port()).toString())) - .build(); - } - -} diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index 259dd788244..dd135a51378 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -3,8 +3,6 @@ package com.yahoo.messagebus.shared; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; - -import java.util.Objects; import java.util.logging.Level; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.IntermediateSessionParams; @@ -27,7 +25,8 @@ public class SharedMessageBus extends AbstractResource { private final MessageBus mbus; public SharedMessageBus(MessageBus mbus) { - this.mbus = Objects.requireNonNull(mbus); + mbus.getClass(); // throws NullPointerException + this.mbus = mbus; } public MessageBus messageBus() { diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index 9512c1a4873..6335cf01d8c 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.messagebus; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.container.jdisc.messagebus.MbusClientProvider; import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; -import com.yahoo.messagebus.MessagebusConfig; -import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import org.junit.Test; import static org.junit.Assert.assertNotNull; @@ -37,15 +30,7 @@ public class MbusClientProviderTest { } private void testClient(SessionConfig config) { - SessionCache cache = new SessionCache(new ContainerMbusConfig.Builder().build(), - new DocumentmanagerConfig.Builder().build(), - new LoadTypeConfig.Builder().build(), - new SlobroksConfig.Builder().build(), - new MessagebusConfig.Builder().build(), - new DocumentProtocolPoliciesConfig.Builder().build(), - new DistributionConfig.Builder().build(), - "test"); - MbusClientProvider p = new MbusClientProvider(cache, config); + MbusClientProvider p = new MbusClientProvider(new SessionCache("dir:src/test/resources/config/clientprovider"), config); assertNotNull(p.get()); p.deconstruct(); } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java index d24acb9d0d8..ef290a070cb 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java @@ -23,6 +23,10 @@ public class ClientTestDriverTestCase { driver = ClientTestDriver.newInstanceWithProtocol(new SimpleProtocol()); assertNotNull(driver); assertTrue(driver.close()); - } + Slobrok slobrok = new Slobrok(); + driver = ClientTestDriver.newInstanceWithExternSlobrok(slobrok.configId()); + assertNotNull(driver); + assertTrue(driver.close()); + } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java index 7b2c1b68549..f6ae2335d12 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java @@ -24,6 +24,11 @@ public class ServerTestDriverTestCase { driver = ServerTestDriver.newInstanceWithProtocol(new SimpleProtocol(), new NonWorkingRequestHandler(), false); assertNotNull(driver); assertTrue(driver.close()); + + Slobrok slobrok = new Slobrok(); + driver = ServerTestDriver.newInstanceWithExternSlobrok(slobrok.configId(), new NonWorkingRequestHandler(), false); + assertNotNull(driver); + assertTrue(driver.close()); } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java index 759509946ce..78e79da4b9f 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.shared; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.*; import com.yahoo.messagebus.jdisc.test.MessageQueue; import com.yahoo.messagebus.jdisc.test.RemoteClient; import com.yahoo.messagebus.jdisc.test.ReplyQueue; -import com.yahoo.messagebus.jdisc.test.TestUtils; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.test.SimpleMessage; @@ -100,7 +98,7 @@ public class SharedDestinationSessionTestCase { RemoteClient client = RemoteClient.newInstanceWithInternSlobrok(true); MessageQueue queue = new MessageQueue(); DestinationSessionParams params = new DestinationSessionParams().setMessageHandler(queue); - SharedDestinationSession session = newDestinationSession(client.slobroksConfig(), params); + SharedDestinationSession session = newDestinationSession(client.slobrokId(), params); Route route = Route.parse(session.connectionSpec()); assertTrue(client.sendMessage(new SimpleMessage("foo").setRoute(route)).isAccepted()); @@ -122,11 +120,11 @@ public class SharedDestinationSessionTestCase { } catch (ListenFailedException e) { fail(); } - return newDestinationSession(TestUtils.configFor(slobrok), new DestinationSessionParams()); + return newDestinationSession(slobrok.configId(), new DestinationSessionParams()); } - private static SharedDestinationSession newDestinationSession(SlobroksConfig slobroksConfig, DestinationSessionParams params) { - RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig); + private static SharedDestinationSession newDestinationSession(String slobrokId, DestinationSessionParams params) { + RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId); MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol()); SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams); SharedDestinationSession session = mbus.newDestinationSession(params); diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java index ca8301d1ea9..87958415149 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.shared; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.*; @@ -9,7 +8,6 @@ import com.yahoo.messagebus.jdisc.test.MessageQueue; import com.yahoo.messagebus.jdisc.test.RemoteClient; import com.yahoo.messagebus.jdisc.test.RemoteServer; import com.yahoo.messagebus.jdisc.test.ReplyQueue; -import com.yahoo.messagebus.jdisc.test.TestUtils; import com.yahoo.messagebus.network.local.LocalNetwork; import com.yahoo.messagebus.network.local.LocalWire; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; @@ -79,7 +77,7 @@ public class SharedIntermediateSessionTestCase { public void requireThatReplyHandlerCanNotBeSet() throws ListenFailedException { Slobrok slobrok = new Slobrok(); try { - newIntermediateSession(TestUtils.configFor(slobrok), + newIntermediateSession(slobrok.configId(), new IntermediateSessionParams().setReplyHandler(new ReplyQueue()), false); fail(); @@ -113,7 +111,7 @@ public class SharedIntermediateSessionTestCase { @Test public void requireThatSessionCanSendMessage() throws InterruptedException { RemoteServer server = RemoteServer.newInstanceWithInternSlobrok(); - SharedIntermediateSession session = newIntermediateSession(server.slobroksConfig(), + SharedIntermediateSession session = newIntermediateSession(server.slobrokId(), new IntermediateSessionParams(), true); ReplyQueue queue = new ReplyQueue(); @@ -136,7 +134,7 @@ public class SharedIntermediateSessionTestCase { RemoteClient client = RemoteClient.newInstanceWithInternSlobrok(true); MessageQueue queue = new MessageQueue(); IntermediateSessionParams params = new IntermediateSessionParams().setMessageHandler(queue); - SharedIntermediateSession session = newIntermediateSession(client.slobroksConfig(), params, true); + SharedIntermediateSession session = newIntermediateSession(client.slobrokId(), params, true); Route route = Route.parse(session.connectionSpec()); assertTrue(client.sendMessage(new SimpleMessage("foo").setRoute(route)).isAccepted()); @@ -158,13 +156,13 @@ public class SharedIntermediateSessionTestCase { } catch (ListenFailedException e) { fail(); } - return newIntermediateSession(TestUtils.configFor(slobrok), new IntermediateSessionParams(), network); + return newIntermediateSession(slobrok.configId(), new IntermediateSessionParams(), network); } - private static SharedIntermediateSession newIntermediateSession(SlobroksConfig slobroksConfig, + private static SharedIntermediateSession newIntermediateSession(String slobrokId, IntermediateSessionParams params, boolean network) { - RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig); + RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId); MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol()); SharedMessageBus mbus = network ? SharedMessageBus.newInstance(mbusParams, netParams) diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java index 7a4a3d17170..1f0966fc961 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.shared; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.Message; @@ -9,7 +8,6 @@ import com.yahoo.messagebus.MessageBusParams; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.jdisc.test.RemoteServer; import com.yahoo.messagebus.jdisc.test.ReplyQueue; -import com.yahoo.messagebus.jdisc.test.TestUtils; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.test.SimpleMessage; @@ -61,7 +59,7 @@ public class SharedSourceSessionTestCase { @Test public void requireThatSessionCanSendMessage() throws InterruptedException { RemoteServer server = RemoteServer.newInstanceWithInternSlobrok(); - SharedSourceSession session = newSourceSession(server.slobroksConfig(), + SharedSourceSession session = newSourceSession(server.slobrokId(), new SourceSessionParams()); ReplyQueue queue = new ReplyQueue(); Message msg = new SimpleMessage("foo").setRoute(Route.parse(server.connectionSpec())); @@ -82,11 +80,11 @@ public class SharedSourceSessionTestCase { } catch (ListenFailedException e) { fail(); } - return newSourceSession(TestUtils.configFor(slobrok), params); + return newSourceSession(slobrok.configId(), params); } - private static SharedSourceSession newSourceSession(SlobroksConfig slobroksConfig, SourceSessionParams params) { - RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig); + private static SharedSourceSession newSourceSession(String slobrokId, SourceSessionParams params) { + RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId); MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol()); SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams); SharedSourceSession session = mbus.newSourceSession(params); diff --git a/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg b/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg diff --git a/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg b/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg diff --git a/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg b/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg diff --git a/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg b/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg diff --git a/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg b/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 2f01f45edaa..02a7028b8ca 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -27,6 +27,13 @@ <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>serviceview</artifactId> <scope>provided</scope> <version>${project.version}</version> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 81c4d7be483..8d9f20a7cee 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -55,7 +55,7 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath); - String getClusterControllerStatus(DeploymentId deployment, String node, String subPath); + String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath); /** * Gets the Vespa logs of the given deployment. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java index 1d23cd52d23..01045eec020 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.time.Instant; import java.util.Map; @@ -20,10 +21,10 @@ public class ApplicationSummary { private final Optional<Instant> lastQueried; private final Optional<Instant> lastWritten; private final Optional<Instant> lastBuilt; - private final Map<ZoneId, Metric> metrics; + private final Map<DeploymentId, Metric> metrics; public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, - Optional<Instant> lastBuilt, Map<ZoneId, Metric> metrics) { + Optional<Instant> lastBuilt, Map<DeploymentId, Metric> metrics) { this.application = Objects.requireNonNull(application); this.lastQueried = Objects.requireNonNull(lastQueried); this.lastWritten = Objects.requireNonNull(lastWritten); @@ -47,7 +48,7 @@ public class ApplicationSummary { return lastBuilt; } - public Map<ZoneId, Metric> metrics() { + public Map<DeploymentId, Metric> metrics() { return metrics; } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 0c05f7d70bb..a0d05588a0d 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -20,6 +20,13 @@ <!-- provided --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>controller-api</artifactId> <version>${project.version}</version> @@ -128,16 +135,6 @@ </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> <groupId>com.auth0</groupId> <artifactId>java-jwt</artifactId> <exclusions> @@ -167,6 +164,14 @@ <scope>compile</scope> </dependency> + <dependency> + <!-- required by java-jwt --> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>${commons.codec.version}</version> + <scope>compile</scope> + </dependency> + <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index a1566a1b5a1..3b6a03a76c9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -66,7 +66,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -89,12 +88,15 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.logging.Level.INFO; @@ -188,11 +190,20 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { + Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate(); return deploy(() -> controller.applications().deploy(id.job(), setTheStage), controller.jobController().run(id).get() .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), - logger); + logger) + .filter(result -> { + // If no tester cert, or deployment failed, propagate original result. + if (testerCertificate.isEmpty() || result != running) + return true; + // If tester cert, ensure real is deployed with the tester cert whose key was successfully deployed. + return controller.jobController().run(id).get().stepStatus(deployTester).get() == succeeded + && testerCertificate.equals(controller.jobController().run(id).get().testerCertificate()); + }); } private Optional<RunStatus> deployTester(RunId id, DualLogger logger) { @@ -249,7 +260,7 @@ public class InternalStepRunner implements StepRunner { case OUT_OF_CAPACITY: logger.log(e.message()); return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant()) - ? Optional.empty() + ? result : Optional.of(outOfCapacity); case INVALID_APPLICATION_PACKAGE: case BAD_REQUEST: @@ -636,6 +647,19 @@ public class InternalStepRunner implements StepRunner { try { controller.jobController().updateVespaLog(id); } + // Hitting a config server which doesn't have this particular app loaded causes a 404. + catch (ConfigServerException e) { + Instant doom = controller.jobController().run(id).get().stepInfo(copyVespaLogs).get().startTime().get() + .plus(Duration.ofMinutes(3)); + if (e.code() == ConfigServerException.ErrorCode.NOT_FOUND && controller.clock().instant().isBefore(doom)) { + logger.log(INFO, "Found no logs, but will retry"); + return Optional.empty(); + } + else { + logger.log(INFO, "Failure getting vespa logs for " + id, e); + return Optional.of(error); + } + } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); return Optional.of(error); @@ -685,6 +709,12 @@ public class InternalStepRunner implements StepRunner { logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e); return Optional.of(error); } + catch (RuntimeException e) { + Instant start = controller.jobController().run(id).get().stepInfo(report).get().startTime().get(); + return (controller.clock().instant().isAfter(start.plusSeconds(180))) + ? Optional.empty() + : Optional.of(error); + } return Optional.of(running); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 25bc21a0076..d9d82b41e71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -230,6 +231,14 @@ public class JobController { return runs(id.application(), id.type()); } + /** Lists the start time of non-redeployment runs of the given job, in order of increasing age. */ + public List<Instant> jobStarts(JobId id) { + return runs(id).descendingMap().values().stream() + .filter(run -> ! run.isRedeployment()) + .map(Run::start) + .collect(toUnmodifiableList()); + } + /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); @@ -359,7 +368,7 @@ public class JobController { List<Lock> locks = new ArrayList<>(); try { // Ensure no step is still running before we finish the run — report depends transitively on all the other steps. - for (Step step : report.allPrerequisites()) + for (Step step : report.allPrerequisites(run(id).get().steps().keySet())) locks.add(curator.lock(id.application(), id.type(), step)); locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails. @@ -440,18 +449,23 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { - start(id, type, versions, JobProfile.of(type)); + start(id, type, versions, false); + } + + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment) { + start(id, type, versions, isRedeployment, JobProfile.of(type)); } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) { + public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile) { locked(id, type, __ -> { Optional<Run> last = last(id, type); if (last.flatMap(run -> active(run.id())).isPresent()) throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile)); + curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile)); metric.jobStarted(newId.job()); }); } @@ -477,6 +491,7 @@ public class JobController { ApplicationVersion.unknown, Optional.empty(), Optional.empty()), + false, JobProfile.development); }); @@ -573,7 +588,7 @@ public class JobController { /** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */ public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException { try (Lock lock = curator.lock(id, type, step)) { - for (Step prerequisite : step.allPrerequisites()) // Check that no prerequisite is still running. + for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running. try (Lock __ = curator.lock(id, type, prerequisite)) { ; } action.accept(new LockedStep(lock, step)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index d2481cd97ad..d93a0133c97 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -30,6 +30,7 @@ public class Run { private final RunId id; private final Map<Step, StepInfo> steps; private final Versions versions; + private final boolean isRedeployment; private final Instant start; private final Optional<Instant> end; private final RunStatus status; @@ -40,12 +41,13 @@ public class Run { private final Optional<X509Certificate> testerCertificate; // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end, + public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; + this.isRedeployment = isRedeployment; this.start = start; this.end = end; this.status = status; @@ -56,10 +58,10 @@ public class Run { this.testerCertificate = testerCertificate; } - public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) { + public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -73,7 +75,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); - return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, + return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } @@ -88,49 +90,49 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(startTime)); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run finished(Instant now) { requireActive(); - return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status, + return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty()); } public Run aborted() { requireActive(); - return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(long lastTestRecord) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } public Run noNodesDownSince(Instant noNodesDownSince) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate); } public Run withSummary(ConvergenceSummary convergenceSummary) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.of(testerCertificate)); } @@ -222,6 +224,11 @@ public class Run { return testerCertificate; } + /** Whether this is a automatic redeployment. */ + public boolean isRedeployment() { + return isRedeployment; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index baba4771370..ce34a021218 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -1,10 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -34,7 +36,7 @@ public enum Step { installTester(false, deployTester), /** Download and deploy the initial real application, for staging tests. */ - deployInitialReal(false, deployTester), + deployInitialReal(false), /** See that the real application has had its nodes converge to the initial state. */ installInitialReal(false, deployInitialReal), @@ -46,7 +48,7 @@ public enum Step { endStagingSetup(false, startStagingSetup), /** Download and deploy real application, restarting services if required. */ - deployReal(false, endStagingSetup, deployTester), + deployReal(false, endStagingSetup), /** See that real application has had its nodes converge to the wanted version and generation. */ installReal(false, deployReal), @@ -72,24 +74,24 @@ public enum Step { private final boolean alwaysRun; private final List<Step> prerequisites; - private final List<Step> allPrerequisites; Step(boolean alwaysRun, Step... prerequisites) { this.alwaysRun = alwaysRun; this.prerequisites = List.of(prerequisites); - this.allPrerequisites = Stream.concat(Stream.of(prerequisites), - Stream.of(prerequisites).flatMap(pre -> pre.allPrerequisites().stream())) - .sorted() - .distinct() - .collect(toUnmodifiableList()); } /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */ public boolean alwaysRun() { return alwaysRun; } - /** Returns all prerequisite steps for this, recursively. */ - public List<Step> allPrerequisites() { - return allPrerequisites; + /** Returns all prerequisite steps for this, including transient ones, in a job profile containing the given steps. */ + public List<Step> allPrerequisites(Collection<Step> among) { + return prerequisites.stream() + .filter(among::contains) + .flatMap(pre -> Stream.concat(Stream.of(pre), + pre.allPrerequisites(among).stream())) + .sorted() + .distinct() + .collect(toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 69e0eb26f16..de087737320 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; @@ -17,7 +18,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.HashMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -78,14 +78,15 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { private ApplicationSummary summaryOf(TenantAndApplicationId application) { var app = applications.requireApplication(application); - var metrics = new HashMap<ZoneId, ApplicationSummary.Metric>(); + var metrics = new HashMap<DeploymentId, ApplicationSummary.Metric>(); for (Instance instance : app.instances().values()) { for (var kv : instance.deployments().entrySet()) { var zone = kv.getKey(); var deploymentMetrics = kv.getValue().metrics(); - metrics.put(zone, new ApplicationSummary.Metric(deploymentMetrics.documentCount(), - deploymentMetrics.queriesPerSecond(), - deploymentMetrics.writesPerSecond())); + metrics.put(new DeploymentId(instance.id(), zone), + new ApplicationSummary.Metric(deploymentMetrics.documentCount(), + deploymentMetrics.queriesPerSecond(), + deploymentMetrics.writesPerSecond())); } } return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), @@ -112,20 +113,29 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } private double updateConfirmedApplicationOwners() { + AtomicInteger attempts = new AtomicInteger(0); + AtomicInteger failures = new AtomicInteger(0); applications() - .withProjectId() - .withProductionDeployment() - .asList() - .stream() - .filter(application -> application.ownershipIssueId().isPresent()) - .forEach(application -> { - IssueId ownershipIssueId = application.ownershipIssueId().get(); - ownershipIssues.getConfirmedOwner(ownershipIssueId).ifPresent(owner -> { - controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.withOwner(owner))); - }); - }); - return 1.0; + .withProjectId() + .withProductionDeployment() + .asList() + .stream() + .filter(application -> application.ownershipIssueId().isPresent()) + .forEach(application -> { + attempts.incrementAndGet(); + IssueId issueId = application.ownershipIssueId().get(); + try { + ownershipIssues.getConfirmedOwner(issueId).ifPresent(owner -> { + controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> + controller().applications().store(lockedApplication.withOwner(owner))); + }); + } + catch (RuntimeException e) { + failures.incrementAndGet(); + log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); + } + }); + return asSuccessFactor(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 91dfed500e3..56bf870c7fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -42,6 +42,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(upgrader); maintainers.addAll(osUpgraders(controller, intervals.osUpgrader)); maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); + maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); maintainers.add(new MetricsReporter(controller, metric)); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9e3da506ca8..40191190eff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import java.util.logging.Level; @@ -62,9 +63,8 @@ public class DeploymentExpirer extends ControllerMaintainer { .map(type -> new JobId(instance, type)); if (jobId.isEmpty()) return false; - return controller().jobController().last(jobId.get()) - .flatMap(Run::end) - .map(end -> end.plus(ttl.get()).isBefore(controller().clock().instant())) + return controller().jobController().jobStarts(jobId.get()).stream().findFirst() + .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant())) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java new file mode 100644 index 00000000000..a69af024b96 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -0,0 +1,92 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; + +/** + * Upgrades instances in manually deployed zones to the system version, at a convenient time. + * + * @author jonmv + */ +public class DeploymentUpgrader extends ControllerMaintainer { + + public DeploymentUpgrader(Controller controller, Duration interval) { + super(controller, interval); + } + + @Override + protected double maintain() { + AtomicInteger attempts = new AtomicInteger(); + AtomicInteger failures = new AtomicInteger(); + Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty()); + for (Application application : controller().applications().readable()) + for (Instance instance : application.instances().values()) + for (Deployment deployment : instance.deployments().values()) + try { + attempts.incrementAndGet(); + JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + if ( ! deployment.zone().environment().isManuallyDeployed()) continue; + if ( ! deployment.version().isBefore(target.targetPlatform())) continue; + if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue; + if ( ! isLikelyNightFor(job)) continue; + + log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); + controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true); + } catch (Exception e) { + failures.incrementAndGet(); + log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + + ": " + Exceptions.toMessageString(e) + ". Retrying in " + + interval()); + } + return asSuccessFactor(attempts.get(), failures.get()); + } + + private boolean isLikelyNightFor(JobId job) { + int hour = hourOf(controller().clock().instant()); + int[] runStarts = controller().jobController().jobStarts(job).stream() + .mapToInt(DeploymentUpgrader::hourOf) + .toArray(); + int localNight = mostLikelyWeeHour(runStarts); + return Math.abs(hour - localNight) <= 1; + } + + static int mostLikelyWeeHour(int[] starts) { + double weight = 1; + double[] buckets = new double[24]; + for (int start : starts) + buckets[start] += weight *= 0.8; // Weight more recent deployments higher. + + int best = -1; + double min = Double.MAX_VALUE; + for (int i = 12; i < 36; i++) { + double sum = 0; + for (int j = -12; j < 12; j++) + sum += buckets[(i + j) % 24] / (Math.abs(j) + 1); + + if (sum < min) { + min = sum; + best = i; + } + } + return (best + 2) % 24; + } + + private static int hourOf(Instant instant) { + return (int) (instant.toEpochMilli() / 3_600_000 % 24); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 85a69b0f338..cd7ce8c3fa6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -86,14 +86,14 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } /** - * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs. + * If it's been four days since the cert has been refreshed, re-trigger all prod deployment jobs. */ private void deployRefreshedCertificates() { var now = clock.instant(); curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); - if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) { + if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index 5942291f2b0..c2addbe9a67 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -47,10 +47,9 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (currentTarget.isEmpty()) return; if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete - controller().upgradeOsIn(cloud, - release.version(currentTarget.get(), now), - release.upgradeBudget(), - false); + Version version = release.version(currentTarget.get(), now); + if (!version.isAfter(currentTarget.get().osVersion().version())) return; + controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false); } private boolean upgradingToNewMajor(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 87527085237..8ffa4823ead 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -83,6 +83,7 @@ class RunSerializer { private static final String endField = "end"; private static final String statusField = "status"; private static final String versionsField = "versions"; + private static final String isRedeploymentField = "isRedeployment"; private static final String platformVersionField = "platform"; private static final String repositoryField = "repository"; private static final String branchField = "branch"; @@ -131,6 +132,7 @@ class RunSerializer { runObject.field(numberField).asLong()), steps, versionsFromSlime(runObject.field(versionsField)), + runObject.field(isRedeploymentField).asBool(), SlimeUtils.instant(runObject.field(startField)), SlimeUtils.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), @@ -177,7 +179,7 @@ class RunSerializer { compileVersion, buildTime, sourceUrl, commit); } - // Don't change this — introduce a separate array instead. + // Don't change this — introduce a separate array instead. private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) { if ( ! summaryArray.valid()) return Optional.empty(); @@ -215,6 +217,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); runObject.setString(jobTypeField, run.id().type().jobName()); + runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d022588c757..c1e6c362c83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -245,7 +245,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -546,28 +546,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (controller.tenants().get(tenantName).isEmpty()) return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); + List<Application> applications = applicationName.isEmpty() ? + controller.applications().asList(tenant) : + controller.applications().getApplication(TenantAndApplicationId.from(tenantName, applicationName.get())) + .map(List::of) + .orElseThrow(() -> new NotExistsException("Application '" + applicationName.get() + "' does not exist")); + Slime slime = new Slime(); Cursor applicationArray = slime.setArray(); - for (Application application : controller.applications().asList(tenant)) { - if (applicationName.map(application.id().application().value()::equals).orElse(true)) { - Cursor applicationObject = applicationArray.addObject(); - applicationObject.setString("tenant", application.id().tenant().value()); - applicationObject.setString("application", application.id().application().value()); - applicationObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), - request.getUri()).toString()); - Cursor instanceArray = applicationObject.setArray("instances"); - for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() - : application.instances().keySet()) { - Cursor instanceObject = instanceArray.addObject(); - instanceObject.setString("instance", instance.value()); - instanceObject.setString("url", withPath("/application/v4" + - "/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value() + - "/instance/" + instance.value(), - request.getUri()).toString()); - } + for (Application application : applications) { + Cursor applicationObject = applicationArray.addObject(); + applicationObject.setString("tenant", application.id().tenant().value()); + applicationObject.setString("application", application.id().application().value()); + applicationObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value(), + request.getUri()).toString()); + Cursor instanceArray = applicationObject.setArray("instances"); + for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() + : application.instances().keySet()) { + Cursor instanceObject = instanceArray.addObject(); + instanceObject.setString("instance", instance.value()); + instanceObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + instance.value(), + request.getUri()).toString()); } } return new SlimeJsonResponse(slime); @@ -1383,11 +1387,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); response.setString("revision", deployment.applicationVersion().id()); - response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); + Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment); + response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli()); controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) - .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); + .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli())); - DeploymentStatus status = controller.jobController().deploymentStatus(application); application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.applicationVersion().source(), response); @@ -1396,18 +1400,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) toSlime(instance.rotations(), instance.rotationStatus(), deployment, response); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) - .ifPresent(stepStatus -> { - JobControllerApiHandlerHelper.applicationVersionToSlime( - response.setObject("applicationVersion"), deployment.applicationVersion()); - if (!status.jobsToRun().containsKey(stepStatus.job().get())) - response.setString("status", "complete"); - else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) - response.setString("status", "pending"); - else response.setString("status", "running"); - }); + if (!deployment.zone().environment().isManuallyDeployed()) { + DeploymentStatus status = controller.jobController().deploymentStatus(application); + JobType.from(controller.system(), deployment.zone()) + .map(type -> new JobId(instance.id(), type)) + .map(status.jobSteps()::get) + .ifPresent(stepStatus -> { + JobControllerApiHandlerHelper.applicationVersionToSlime( + response.setObject("applicationVersion"), deployment.applicationVersion()); + if (!status.jobsToRun().containsKey(stepStatus.job().get())) + response.setString("status", "complete"); + else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) + response.setString("status", "pending"); + else response.setString("status", "running"); + }); + } } response.setDouble("quota", deployment.quota().rate()); @@ -1435,6 +1442,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } + private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { + return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get())) + .stream().findFirst().orElse(deployment.at()); + } + private void toSlime(ApplicationVersion applicationVersion, Cursor object) { if ( ! applicationVersion.isUnknown()) { object.setLong("buildNumber", applicationVersion.buildNumber().getAsLong()); @@ -1666,9 +1678,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); - if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) { - String[] parts = restPath.split("/status/"); - String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, parts[0], parts[1]); + if (restPath.contains("/status/")) { + String[] parts = restPath.split("/status/", 2); + String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, serviceName, parts[0], parts[1]); return new HtmlResponse(result); } @@ -2079,12 +2091,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.serviceRegistry().roleService().getTenantRole(tenant.name()), cloudTenant.tenantSecretStores()); - var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name()); - var usedQuota = applications.stream() - .map(Application::quotaUsage) - .reduce(QuotaUsage.none, QuotaUsage::add); + try { + var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name()); + var usedQuota = applications.stream() + .map(Application::quotaUsage) + .reduce(QuotaUsage.none, QuotaUsage::add); - toSlime(tenantQuota, usedQuota, object.setObject("quota")); + toSlime(tenantQuota, usedQuota, object.setObject("quota")); + } catch (Exception e) { + log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e))); + } cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role)); @@ -2187,9 +2203,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) { Optional<Instant> lastDev = applications.stream() .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.deployments().values().stream()) - .filter(deployment -> deployment.zone().environment() == Environment.dev) - .map(Deployment::at) + .flatMap(instance -> instance.deployments().values().stream() + .filter(deployment -> deployment.zone().environment() == Environment.dev) + .map(deployment -> lastDeploymentStart(instance.id(), deployment))) .max(Comparator.naturalOrder()) .or(() -> applications.stream() .flatMap(application -> application.instances().values().stream()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index d246b20f2fe..eb9b0e03f3d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -40,10 +40,12 @@ import java.time.Instant; import java.time.format.TextStyle; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; @@ -84,7 +86,7 @@ class JobControllerApiHandlerHelper { Cursor jobObject = jobsArray.addObject(); jobObject.setString("jobName", job.type().jobName()); - toSlime(jobObject.setArray("runs"), runs, baseUriForJobs); + toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs); }); return new SlimeJsonResponse(slime); @@ -117,11 +119,19 @@ class JobControllerApiHandlerHelper { } /** Returns a response with the runs for the given job type. */ - static HttpResponse runResponse(Map<RunId, Run> runs, URI baseUriForJobType) { + static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) { Slime slime = new Slime(); Cursor cursor = slime.setObject(); - runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); + // TODO (freva): Remove after console migrated to use new format + if (limitStr.isEmpty()) + runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType)); + else { + int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); + toSlime(cursor.setArray("runs"), runs.values().stream() + .sorted(Comparator.comparing((Run run) -> run.id().number()).reversed()) + .collect(Collectors.toUnmodifiableList()), limit, baseUriForJobType); + } return new SlimeJsonResponse(slime); } @@ -377,7 +387,7 @@ class JobControllerApiHandlerHelper { toSlime(runObject.setObject("versions"), versions); } - toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob); + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); }); } @@ -428,8 +438,8 @@ class JobControllerApiHandlerHelper { return Optional.of(versions.get(i)); } - private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) { - runs.stream().limit(10).forEach(run -> { + private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) { + runs.stream().limit(limit).forEach(run -> { Cursor runObject = runsArray.addObject(); runObject.setLong("id", run.id().number()); runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 4951f2ba33b..c13a9b42382 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -113,7 +113,7 @@ public class Badges { List<Run> runs = status.runs().descendingMap().values().stream() .filter(Run::hasEnded) .skip(1) - .limit(length + (lastTriggered.hasEnded() ? 0 : 1)) + .limit(length) .collect(toList()); boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 14244d7bdda..d8bddac4187 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -58,6 +57,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -383,28 +383,81 @@ public class InternalStepRunnerTest { @Test public void vespaLogIsCopied() { + // Tests fail. We should get logs. This fails too, on the first attempt. RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.ERROR); - tester.configServer().setLogStream(vespaLog); + tester.configServer().setLogStream(() -> { throw new ConfigServerException(ConfigServerException.ErrorCode.NOT_FOUND, "404", "context"); }); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.copyVespaLogs)); assertTestLogEntries(id, Step.copyVespaLogs, - new LogEntry(lastId + 2, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 2, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 4, tester.clock().instant(), info, + "Found no logs, but will retry")); + + // Config servers now provide the log, and we get it. + tester.configServer().setLogStream(() -> vespaLog); + tester.runner().run(); + assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + assertTestLogEntries(id, Step.copyVespaLogs, + new LogEntry(lastId + 2, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 4, tester.clock().instant(), info, + "Found no logs, but will retry"), + new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 3, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 6, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 4, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 7, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, + new LogEntry(lastId + 8, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); } @Test + public void realDeploymentRequiresForTesterCert() { + tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); + var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); + ZoneId testZone = JobType.systemTest.zone(tester.controller().system()); + RunId id = app.newRun(JobType.systemTest); + tester.configServer().throwOnPrepare(instanceId -> { + if (instanceId.instance().isTester()) + throw new ConfigServerException(ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY, "provisioning", "deploy tester"); + }); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); + + List<X509Certificate> oldTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); + X509Certificate oldCert = tester.jobs().run(id).get().testerCertificate().get(); + oldTrusted.add(oldCert); + assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + + tester.configServer().throwOnNextPrepare(null); + tester.runner().run(); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); + assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); + + List<X509Certificate> newTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); + X509Certificate newCert = tester.jobs().run(id).get().testerCertificate().get(); + newTrusted.add(newCert); + assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertNotEquals(oldCert, newCert); + } + + @Test public void certificateTimeoutAbortsJob() { tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index dca79c43afe..92c8cbc4889 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -63,6 +63,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -93,9 +98,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private List<ProtonMetrics> protonMetrics; private Version lastPrepareVersion = null; - private RuntimeException prepareException = null; + private Consumer<ApplicationId> prepareException = null; private ConfigChangeActions configChangeActions = null; - private String log = "INFO - All good"; + private Supplier<String> log = () -> "INFO - All good"; @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { @@ -205,9 +210,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Optional.ofNullable(lastPrepareVersion); } + /** Sets a function that may throw, determined by app id. */ + public void throwOnPrepare(Consumer<ApplicationId> prepareThrower) { + this.prepareException = prepareThrower; + } + /** The exception to throw on the next prepare run, or null to continue normally */ public void throwOnNextPrepare(RuntimeException prepareException) { - this.prepareException = prepareException; + this.prepareException = prepareException == null ? null : id -> { this.prepareException = null; throw prepareException; }; } /** Set version for an application in a given zone */ @@ -366,11 +376,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentData deployment) { lastPrepareVersion = deployment.platform(); - if (prepareException != null) { - RuntimeException prepareException = this.prepareException; - this.prepareException = null; - throw prepareException; - } + if (prepareException != null) + prepareException.accept(ApplicationId.from(deployment.instance().tenant(), + deployment.instance().application(), + deployment.instance().instance())); DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); @@ -523,7 +532,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public String getClusterControllerStatus(DeploymentId deployment, String node, String subPath) { + public String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath) { return "<h1>OK</h1>"; } @@ -554,7 +563,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters) { - return new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)); + return new ByteArrayInputStream(log.get().getBytes(StandardCharsets.UTF_8)); } @Override @@ -562,7 +571,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return new ProxyResponse("{\"path\":\"" + path + "\"}", "application/json", 200); } - public void setLogStream(String log) { + public void setLogStream(Supplier<String> log) { this.log = log; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index ffc82f90ad4..31f8aaf9e2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -70,7 +70,9 @@ public class DeploymentExpirerTest { assertEquals(1, permanentDeployments(prodApp.instance())); // Dev application expires when enough time has passed since most recent attempt + // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); + tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java new file mode 100644 index 00000000000..7a8f775e8b1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -0,0 +1,101 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour; +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author jonmv + */ +public class DeploymentUpgraderTest { + + private final DeploymentTester tester = new DeploymentTester(); + + @Test + public void testDeploymentUpgrading() { + ZoneId devZone = ZoneId.from(Environment.dev, RegionName.from("us-east-1")); + DeploymentUpgrader upgrader = new DeploymentUpgrader(tester.controller(), Duration.ofDays(1)); + var devApp = tester.newDeploymentContext("tenant1", "app1", "default"); + var prodApp = tester.newDeploymentContext("tenant2", "app2", "default"); + + ApplicationPackage appPackage = new ApplicationPackageBuilder().region("us-west-1").build(); + Version systemVersion = tester.controller().readSystemVersion(); + Instant start = tester.clock().instant().truncatedTo(MILLIS); + + devApp.runJob(devUsEast1, appPackage); + prodApp.submit(appPackage).deploy(); + assertEquals(systemVersion, tester.jobs().last(devApp.instanceId(), devUsEast1).get().versions().targetPlatform()); + assertEquals(systemVersion, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().versions().targetPlatform()); + + // Not upgraded initially + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // Not upgraded immediately after system upgrades + tester.controllerTester().upgradeSystem(new Version(7, 8, 9)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 14 hours pass, but not upgraded before a day has passed since last deployment + tester.clock().advance(Duration.ofHours(14)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 35 hours pass, but not upgraded since it's not likely in the middle of the night + tester.clock().advance(Duration.ofHours(21)); + upgrader.maintain(); + assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + + // 38 hours pass, and the dev deployment, only, is upgraded + tester.clock().advance(Duration.ofHours(3)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + assertTrue(tester.jobs().last(devApp.instanceId(), devUsEast1).get().isRedeployment()); + assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start()); + devApp.runJob(devUsEast1); + + // After the upgrade, the dev app is mostly (re)deployed to at night, but this doesn't affect what is likely the night. + tester.controllerTester().upgradeSystem(new Version(7, 9, 11)); + tester.clock().advance(Duration.ofHours(48)); + upgrader.maintain(); + assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start()); + } + + @Test + public void testNight() { + assertEquals(16, mostLikelyWeeHour(new int[]{ 0, 1, 2, 3, 4, 5, 6 })); + assertEquals(14, mostLikelyWeeHour(new int[]{ 22, 23, 0, 1, 2, 3, 4 })); + assertEquals(18, mostLikelyWeeHour(new int[]{ 6, 5, 4, 3, 2, 1, 0 })); + assertEquals(20, mostLikelyWeeHour(new int[]{ 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 11 })); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index ce219b8beed..3ed26b4fb6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -82,7 +82,7 @@ public class EndpointCertificateMaintainerTest { } @Test - public void refreshed_certificate_is_deployed_after_one_week() { + public void refreshed_certificate_is_deployed_after_four_days() { var appId = ApplicationId.from("tenant", "application", "default"); DeploymentTester deploymentTester = new DeploymentTester(tester); @@ -107,7 +107,7 @@ public class EndpointCertificateMaintainerTest { maintainer.maintain(); - tester.clock().advance(Duration.ofDays(8)); + tester.clock().advance(Duration.ofDays(4)); deploymentContext.assertNotRunning(productionUsWest1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index f677cc079cc..023c5671b60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -131,8 +131,8 @@ public class JobRunnerTest { Map<Step, Status> steps = run.get().stepStatuses(); runner.maintain(); assertEquals(steps, run.get().stepStatuses()); - assertEquals(List.of(deployTester), run.get().readySteps()); - assertStepsWithStartTime(run.get(), deployTester); + assertEquals(List.of(deployTester, deployReal), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal); outcomes.put(deployTester, running); runner.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index d87dfcfa315..d80a8ce1152 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -132,6 +132,16 @@ public class OsUpgradeSchedulerTest { assertEquals("Target is unchanged as not enough time has passed", version1, target1.osVersion().version()); assertEquals("Target is not re-scheduled", target0.scheduledAt(), target1.scheduledAt()); + + // A newer version is triggered manually + Version version3 = Version.fromString("8.3"); + tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); + + // Enough time passes for stable version to be promoted. Nothing happens as stable is now before the manually + // triggered version + tester.clock().advance(Duration.ofDays(14).plus(Duration.ofSeconds(1))); + scheduler.maintain(); + assertEquals(version3, tester.controller().osVersionTarget(cloud).get().osVersion().version()); } private static ZoneApi zone(String id, CloudName cloud) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 6e12373640d..03a050db74e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -149,7 +149,7 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); - Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production); + Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 3fa593fcf64..a01097cfcb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -225,6 +225,10 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), new File("application-list.json")); + // GET tenant application instances for application that does not exist + tester.assertResponse(request("/application/v4/tenant/tenant1/application/fake-app/instance/", GET).userIdentity(USER_ID), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Application 'fake-app' does not exist\"}", 404); + // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), new File("application-list.json")); @@ -801,7 +805,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET system test job overview. tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET) - .userIdentity(USER_ID), + .userIdentity(USER_ID).properties(Map.of("limit", "100")), new File("system-test-job.json")); // GET system test run 1 details. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 1be7f16e85f..72295497c03 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -131,14 +131,14 @@ public class JobControllerApiHandlerHelperTest { // Only us-east-3 is verified, on revision1. // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2, // one success from revision2 to revision3 and two failures from revision1 to revision3. - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json"); var userApp = tester.newDeploymentContext(app.instanceId().tenant().value(), app.instanceId().application().value(), "user"); userApp.runJob(devAwsUsEast2a, applicationPackage); - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } @@ -151,10 +151,10 @@ public class JobControllerApiHandlerHelperTest { ZoneId zone = JobType.devUsEast1.zone(tester.controller().system()); tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); - tester.configServer().setLogStream("1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); + tester.configServer().setLogStream(() -> "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json"); - tester.configServer().setLogStream("Nope, this won't be logged"); + tester.configServer().setLogStream(() -> "Nope, this won't be logged"); tester.configServer().convergeServices(app.instanceId(), zone); tester.runner().run(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 3234333c092..c0988e8c301 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -1,69 +1,119 @@ { - "1": { - "id": 1, - "status": "success", - "start": "(ignore)", - "end": "(ignore)", - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + "runs": [ + { + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", + "start": "(ignore)", + "status": "running", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" + "steps": [ + { + "name": "deployTester", + "status": "unfinished" + }, + { + "name": "installTester", + "status": "unfinished" + }, + { + "name": "deployReal", + "status": "unfinished" + }, + { + "name": "installReal", + "status": "unfinished" + }, + { + "name": "startTests", + "status": "unfinished" + }, + { + "name": "endTests", + "status": "unfinished" + }, + { + "name": "copyVespaLogs", + "status": "unfinished" + }, + { + "name": "deactivateReal", + "status": "unfinished" + }, + { + "name": "deactivateTester", + "status": "unfinished" + }, + { + "name": "report", + "status": "unfinished" + } + ] }, - "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", - "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "copyVespaLogs": "succeeded", - "deactivateReal": "succeeded", - "deactivateTester": "succeeded", - "report": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" - }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" - }, - "2": { - "id": 2, - "status": "running", - "start": "(ignore)", - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "1.0.4-commit1", - "build": 4, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", - "deployReal": "unfinished", - "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "copyVespaLogs": "unfinished", - "deactivateReal": "unfinished", - "deactivateTester": "unfinished", - "report": "unfinished" - }, - "tasks": {}, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2" - } + "steps": [ + { + "name": "deployTester", + "status": "succeeded" + }, + { + "name": "installTester", + "status": "succeeded" + }, + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "startTests", + "status": "succeeded" + }, + { + "name": "endTests", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + }, + { + "name": "deactivateReal", + "status": "succeeded" + }, + { + "name": "deactivateTester", + "status": "succeeded" + }, + { + "name": "report", + "status": "succeeded" + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index e906df94023..668baa50cc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -43,6 +43,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "DeploymentUpgrader" + }, + { "name": "EndpointCertificateMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 766a84ca33c..63474ebb7c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -53,6 +53,8 @@ public class BadgeApiTest extends ControllerContainerTest { tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), Files.readString(Paths.get(responseFiles + "overview.svg")), 200); + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), + Files.readString(Paths.get(responseFiles + "single-running.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); @@ -66,6 +68,9 @@ public class BadgeApiTest extends ControllerContainerTest { tester.serviceRegistry().clock().advance(Duration.ofSeconds(1)); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history2.svg")), 200); + + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), + Files.readString(Paths.get(responseFiles + "single-done.svg")), 200); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg new file mode 100644 index 00000000000..3bcbed97499 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg @@ -0,0 +1,81 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f244' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='294.82606' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='294.82606' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text> + <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text> + </g> +</svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg new file mode 100644 index 00000000000..27e967f8e46 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg @@ -0,0 +1,81 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f244' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='294.82606' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#run-on-failure)'/> + <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='294.82606' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text> + <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text> + </g> +</svg> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index f034c9260e4..152a0d3eef3 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -31,7 +31,11 @@ endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) + if (VESPA_OS_DISTRO_NAME STREQUAL "CentOS Stream") + set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) + else() + set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE) + endif() endfunction() function(setup_vespa_default_build_settings_rocky_8_4) diff --git a/dist/vespa.spec b/dist/vespa.spec index 7bee7e2a8d0..d8e3653af39 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -102,7 +102,12 @@ BuildRequires: cmake >= 3.11.4-3 BuildRequires: libarchive %endif %define _command_cmake cmake +%global _centos_stream %(grep -qs '^NAME="CentOS Stream"' /etc/os-release && echo 1 || echo 0) +%if 0%{?_centos_stream} +BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) +%else BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12) +%endif %else BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) %endif @@ -226,7 +231,11 @@ Requires: vespa-valgrind >= 3.17.0-1 %endif %if 0%{?el8} %if 0%{?centos} || 0%{?rocky} +%if 0%{?_centos_stream} +%define _vespa_llvm_version 12 +%else %define _vespa_llvm_version 11 +%endif %else %define _vespa_llvm_version 10 %endif @@ -358,7 +367,11 @@ Requires: openssl-libs %endif %if 0%{?el8} %if 0%{?centos} || 0%{?rocky} +%if 0%{?_centos_stream} +Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) +%else Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12) +%endif %else Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) %endif @@ -781,7 +794,6 @@ fi %{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar %{_prefix}/lib/jars/container-apache-http-client-bundle-jar-with-dependencies.jar %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar -%{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar %{_prefix}/lib/jars/defaults-jar-with-dependencies.jar diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java index 268a40f04c9..c508e7da61d 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java @@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; /** - * @author Einar M R Rosenvinge + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> */ public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentProcessingHandlerTestBase { diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java index c437261224d..05009f484c8 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java @@ -5,7 +5,6 @@ import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.core.document.ContainerDocumentConfig; -import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.docproc.CallStack; @@ -14,15 +13,12 @@ import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.application.ContainerBuilder; -import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.jdisc.MbusClient; @@ -30,8 +26,6 @@ import com.yahoo.messagebus.jdisc.test.RemoteServer; import com.yahoo.messagebus.jdisc.test.ServerTestDriver; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import org.junit.After; import org.junit.Before; @@ -58,11 +52,8 @@ public abstract class DocumentProcessingHandlerTestBase { driver = ServerTestDriver.newInactiveInstanceWithProtocol(protocol, true); - sessionCache = new SessionCache(new ContainerMbusConfig.Builder().build(), - driver.client().slobroksConfig(), - new MessagebusConfig.Builder().build(), - "test", - protocol); + sessionCache = + new SessionCache("raw:", driver.client().slobrokId(), "test", "raw:", null, "raw:", documentTypeManager); ContainerBuilder builder = driver.parent().newContainerBuilder(); ComponentRegistry<DocprocService> registry = new ComponentRegistry<>(); @@ -111,7 +102,7 @@ public abstract class DocumentProcessingHandlerTestBase { resource.release(); } - remoteServer = RemoteServer.newInstance("foobar", protocol); + remoteServer = RemoteServer.newInstance(driver.client().slobrokId(), "foobar", protocol); } @After diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 8bfa85d8b16..e922d878dd7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -20,6 +20,12 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> @@ -58,10 +64,6 @@ <artifactId>airline</artifactId> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9f78056d15b..a3e6fff410e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -137,12 +137,6 @@ public class Flags { "Takes effect on the next suspension request to the Orchestrator.", APPLICATION_ID); - public static final UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag( - "encrypt-disk", false, - List.of("hakonhall"), "2021-05-05", "2021-08-05", - "Allow migrating an unencrypted data partition to being encrypted.", - "Takes effect on next host-admin tick."); - public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag( "encrypt-dirty-disk", false, List.of("hakonhall"), "2021-05-14", "2021-08-05", @@ -292,6 +286,12 @@ public class Flags { "List of applications where encryption of their host should be deferred", "Takes effect on next run of HostEncrypter"); + public static final UnboundBooleanFlag PODMAN3 = defineFeatureFlag( + "podman3", false, + List.of("mpolden"), "2021-07-05", "2021-09-01", + "Whether to use Podman 3 on supported hosts", + "Takes effect on host-admin restart"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 1f85658430f..2a8ec1b9bb9 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -31,22 +31,21 @@ <artifactId>slf4j-api</artifactId> <scope>provided</scope> </dependency> - - <!-- compile scope --> + <!-- Apache client artifacts are provided by the jdisc container and are therefore scoped as such --> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents.client5</groupId> <artifactId>httpclient5</artifactId> - <scope>compile</scope> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java index 896799d4492..9c6e698a154 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java @@ -45,14 +45,10 @@ public final class NormalizeExpression extends Expression { } @Override - public boolean equals(Object obj) { - if (!(obj instanceof NormalizeExpression)) { - return false; - } - NormalizeExpression rhs = (NormalizeExpression)obj; - if (linguistics != rhs.linguistics) { - return false; - } + public boolean equals(Object o) { + if (!(o instanceof NormalizeExpression)) return false; + NormalizeExpression other = (NormalizeExpression)o; + if (linguistics != other.linguistics) return false; return true; } @@ -60,4 +56,5 @@ public final class NormalizeExpression extends Expression { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java index e9a0d0253e1..223c8191186 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -90,4 +90,5 @@ public class ExpressionTestCase { assertEquals(foo.hashCode(), bar.hashCode()); assertEquals(foo, bar); } + } diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 9f4572736c8..0c89872aa46 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -24,8 +24,20 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.amazonaws</groupId> diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp index 9259f992d6c..9ff352abcf5 100644 --- a/messagebus/src/tests/targetpool/targetpool.cpp +++ b/messagebus/src/tests/targetpool/targetpool.cpp @@ -37,7 +37,7 @@ TEST("targetpool_test") { FRT_Supervisor & orb = server.supervisor(); std::unique_ptr<PoolTimer> ptr(new PoolTimer()); PoolTimer &timer = *ptr; - RPCTargetPool pool(std::move(ptr), 0.666); + RPCTargetPool pool(std::move(ptr), 0.666, 1 ); // Assert that all connections expire. RPCTarget::SP target; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index fbe00275626..41298c28a88 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -133,7 +133,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)), _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)), _requestedPort(params.getListenPort()), - _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())), + _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs(), params.getNumRpcTargets())), _targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)), _servicePool(std::make_unique<RPCServicePool>(*_mirror, 4_Ki)), _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 64_Ki)), diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp index b42ac47e54d..cfb1e120a2c 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp @@ -4,23 +4,45 @@ namespace mbus { -RPCTargetPool::Entry::Entry(RPCTarget::SP target, uint64_t lastUse) : - _target(target), - _lastUse(lastUse) +RPCTargetPool::Entry::Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse) + : _targets(std::move(targets)), + _lastUse(lastUse), + _next(0) { } -RPCTargetPool::RPCTargetPool(double expireSecs) : - _lock(), - _targets(), - _timer(new SteadyTimer()), - _expireMillis(static_cast<uint64_t>(expireSecs * 1000)) +RPCTarget::SP +RPCTargetPool::Entry::getTarget(const LockGuard &, uint64_t now) { + if (_next >= _targets.size()) { + _next = 0; + } + RPCTarget::SP target = _targets[_next++]; + if ( ! target->isValid()) { + return RPCTarget::SP(); + } + _lastUse = now; + return target; +} + +bool +RPCTargetPool::Entry::inUse(const LockGuard &) const { + for (const auto & target : _targets) { + if (target.use_count() > 1) { + return true; + } + } + return false; +} + +RPCTargetPool::RPCTargetPool(double expireSecs, size_t numTargetsPerSpec) + : RPCTargetPool(std::make_unique<SteadyTimer>(), expireSecs, numTargetsPerSpec) { } -RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs) : +RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec) : _lock(), _targets(), _timer(std::move(timer)), - _expireMillis(static_cast<uint64_t>(expireSecs * 1000)) + _expireMillis(static_cast<uint64_t>(expireSecs * 1000)), + _numTargetsPerSpec(numTargetsPerSpec) { } RPCTargetPool::~RPCTargetPool() @@ -35,21 +57,12 @@ RPCTargetPool::flushTargets(bool force) LockGuard guard(_lock); TargetMap::iterator it = _targets.begin(); while (it != _targets.end()) { - Entry &entry = it->second; - if (entry._target.get() != nullptr) { - if (entry._target.use_count() > 1) { - entry._lastUse = currentTime; - ++it; - continue; // someone is using this - } - if (!force) { - if (entry._lastUse + _expireMillis > currentTime) { - ++it; - continue; // not sufficiently idle - } - } + const Entry &entry = it->second; + if ( ! entry.inUse(guard) && (force || ((entry.lastUse() + _expireMillis) < currentTime))) { + _targets.erase(it++); // postfix increment to move the iterator + } else { + ++it; } - _targets.erase(it++); // postfix increment to move the iterator } } @@ -68,16 +81,19 @@ RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address) LockGuard guard(_lock); auto it = _targets.find(spec); if (it != _targets.end()) { - Entry &entry = it->second; - if (entry._target->isValid()) { - entry._lastUse = currentTime; - return entry._target; + RPCTarget::SP target = it->second.getTarget(guard, currentTime); + if (target) { + return target; } _targets.erase(it); } - auto ret = std::make_shared<RPCTarget>(spec, orb); - _targets.insert(TargetMap::value_type(spec, Entry(ret, currentTime))); - return ret; + std::vector<RPCTarget::SP> targets; + targets.reserve(_numTargetsPerSpec); + for (size_t i(0); i < _numTargetsPerSpec; i++) { + targets.push_back(std::make_shared<RPCTarget>(spec, orb)); + } + _targets.insert(TargetMap::value_type(spec, Entry(std::move(targets), currentTime))); + return _targets.find(spec)->second.getTarget(guard, currentTime); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h index bc9e1a4b19f..2a6fb75d511 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h @@ -16,24 +16,30 @@ namespace mbus { */ class RPCTargetPool { private: + using LockGuard = std::lock_guard<std::mutex>; /** * Implements a helper class holds the necessary reference and token counter * for a JRT target to keep connections open as long as they get used from * time to time. */ - struct Entry { - RPCTarget::SP _target; - uint64_t _lastUse; - - Entry(RPCTarget::SP target, uint64_t lastUse); + class Entry { + public: + Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse); + RPCTarget::SP getTarget(const LockGuard & guard, uint64_t now); + uint64_t lastUse() const { return _lastUse; } + bool inUse(const LockGuard & guard) const; + private: + std::vector<RPCTarget::SP> _targets; + uint64_t _lastUse; + size_t _next; }; using TargetMap = std::map<string, Entry>; - using LockGuard = std::lock_guard<std::mutex>; std::mutex _lock; TargetMap _targets; ITimer::UP _timer; uint64_t _expireMillis; + size_t _numTargetsPerSpec; public: RPCTargetPool(const RPCTargetPool &) = delete; @@ -46,7 +52,7 @@ public: * @param expireSecs The number of seconds until an idle connection is * closed. */ - RPCTargetPool(double expireSecs); + RPCTargetPool(double expireSecs, size_t numTargetsPerSpec); /** * Constructs a new instance of this class, using the given {@link Timer} @@ -57,7 +63,7 @@ public: * @param expireSecs The number of seconds until an idle connection is * closed. */ - RPCTargetPool(ITimer::UP timer, double expireSecs); + RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec); /** * Destructor. Frees any allocated resources. diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index a7579aeb2ea..19b545df616 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -60,6 +60,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-core</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java index 58ff1af4681..75069fa815e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java @@ -25,7 +25,7 @@ public interface ContainerEngine { /** Start a created container */ void startContainer(NodeAgentContext context); - /** Update an existing container */ + /** Update an existing container with new resources */ void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources); /** Remove given container. The container will be stopped if necessary */ @@ -34,25 +34,25 @@ public interface ContainerEngine { /** Get container for given context */ Optional<Container> getContainer(NodeAgentContext context); - /** List containers managed by this */ + /** Returns all containers known by this */ List<PartialContainer> listContainers(TaskContext context); /** Returns the network interface used by container in given context */ String networkInterface(NodeAgentContext context); - /** Executes a command in inside container as root user, throws on non-zero exit code */ + /** Execute command inside container as root. Ignores non-zero exit code */ CommandResult executeAsRoot(NodeAgentContext context, Duration timeout, String... command); - /** Executes a command in inside containers network namespace, throws on non-zero exit code */ + /** Execute command inside the container's network namespace. Throws on non-zero exit code */ CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command); - /** Download giving image */ + /** Download given image */ void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials); /** Returns whether given image is already downloaded */ boolean hasImage(TaskContext context, DockerImage image); - /** Remove given image */ + /** Remove image by id */ void removeImage(TaskContext context, String id); /** Returns images available in this */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 444e92e793f..49216ddfdaf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -70,17 +70,16 @@ public class ContainerOperations { return executeCommandInContainerAsRoot(context, CommandLine.DEFAULT_TIMEOUT.toSeconds(), command); } - /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */ + /** Execute command inside container identified by given context. Does NOT throw on non-zero exit code */ public CommandResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) { return containerEngine.executeAsRoot(context, Duration.ofSeconds(timeoutSeconds), command); } - /** Executes a command in inside containers network namespace, throws on non-zero exit code */ + /** Execute command in inside containers network namespace, identified by given context. Throws on non-zero exit code */ public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) { return containerEngine.executeInNetworkNamespace(context, command); } - /** Resume node. Resuming a node means that it is ready to receive traffic */ public String resumeNode(NodeAgentContext context) { return executeNodeCtlInContainer(context, "resume"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java index c3be670c009..f960201b2a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java @@ -104,8 +104,8 @@ public class PartialContainer { stopped, paused, exited, - removing; - + removing, + stopping; public boolean isRunning() { return this == running; @@ -121,6 +121,7 @@ public class PartialContainer { case "paused": return paused; case "exited": return exited; case "removing": return removing; + case "stopping": return stopping; } throw new IllegalArgumentException("Invalid state '" + state + "'"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java index bb33dfee663..37601055d00 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java @@ -40,11 +40,10 @@ import java.util.stream.Stream; * <li>Deleting a tag of an image with multiple tags will only remove the tag, the image with the * remaining tags will remain</li> * <li>Deleting the last tag of an image will delete the entire image.</li> - * <li>Image cannot be deleted if:</li> - * <ol> - * <li>It has 1 or more children</li> - * <li>A container uses it</li> - * </ol> + * <li>Image cannot be deleted if: + * <p>- It has 1 or more children + * <p>- A container uses it + * </li> * </ol> * * @author freva diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java index 2a01a5ebcb4..ba9ba80ccb7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; @@ -57,16 +56,6 @@ public abstract class YumCommand<T extends YumCommand<T>> { public abstract boolean converge(TaskContext context); - /** Returns the version of Yum itself */ - protected final Version version(TaskContext context) { - return terminal.newCommandLine(context).add("yum", "--version") - .executeSilently() - .getOutputLinesStream() - .findFirst() - .map(Version::fromString).orElseThrow(() -> new IllegalStateException("Failed to detect Yum version")); - } - - public static class GenericYumCommand extends YumCommand<GenericYumCommand> { private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$"); @@ -111,11 +100,10 @@ public abstract class YumCommand<T extends YumCommand<T>> { if (yumCommand == CommandType.remove) if (packages.stream().noneMatch(pkg -> isInstalled(context, pkg))) return false; - Version yumVersion = version(context); CommandLine commandLine = terminal.newCommandLine(context); commandLine.add("yum", yumCommand.name()); addParametersToCommandLine(commandLine); - commandLine.add(packages.stream().map(pkg -> pkg.toName(yumVersion)).collect(Collectors.toList())); + commandLine.add(packages.stream().map(pkg -> pkg.toName()).collect(Collectors.toList())); // There's no way to figure out whether a yum command would have been a no-op. // Therefore, run the command and parse the output to decide. @@ -168,15 +156,10 @@ public abstract class YumCommand<T extends YumCommand<T>> { @Override public boolean converge(TaskContext context) { - Version yumVersion = version(context); - String targetVersionLockName = yumPackage.toVersionLockName(yumVersion); + String targetVersionLockName = yumPackage.toVersionLockName(); List<String> command = new ArrayList<>(4); command.add("yum"); - // Using --quiet on Yum 4 always results in an empty list, even if locks exist... - if (yumVersion.getMajor() < 4) { - command.add("--quiet"); - } command.add("versionlock"); command.add("list"); @@ -193,7 +176,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { if (packageName.getName().equals(yumPackage.getName())) { // If existing lock doesn't exactly match the full package name, // it means it's locked to another version and we must remove that lock. - String versionLockName = packageName.toVersionLockName(yumVersion); + String versionLockName = packageName.toVersionLockName(); if (versionLockName.equals(targetVersionLockName)) { return true; } else { @@ -231,7 +214,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { var installCommand = terminal.newCommandLine(context).add("yum", "install"); addParametersToCommandLine(installCommand); - installCommand.add(yumPackage.toName(yumVersion)); + installCommand.add(yumPackage.toName()); String output = installCommand.executeSilently().getUntrimmedOutput(); @@ -240,7 +223,7 @@ public abstract class YumCommand<T extends YumCommand<T>> { // case 3. var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade"); addParametersToCommandLine(upgradeCommand); - upgradeCommand.add(yumPackage.toName(yumVersion)).execute(); + upgradeCommand.add(yumPackage.toName()).execute(); modified = true; } else { // case 2. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java index abe5dc1cbb3..5b32f50ad49 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.yum; import com.google.common.base.Strings; -import com.yahoo.component.Version; import java.util.Arrays; import java.util.Objects; @@ -12,7 +11,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * YUM package name. + * A YUM/DNF package name. * * <p>From yum(8): YUM package names are used with install, update, remove, list, info etc * with any of the following as well as globs of any of the following, with any of the @@ -42,9 +41,10 @@ public class YumPackageName { Arrays.stream(Architecture.values()).map(Architecture::name).collect(Collectors.joining("|")); private static final Pattern ARCHITECTURE_PATTERN = Pattern.compile("\\.(" + ARCHITECTURES_OR + "|\\*)$"); private static final Pattern EPOCH_PATTERN = Pattern.compile("^((.+)-)?([0-9]+)$"); - private static final Pattern NAME_VER_REL_PATTERN = Pattern.compile("^((.+)-)?" + - "([+()a-z0-9._]*[0-9][a-z0-9._]*)-" + // ver contains at least one digit - "([+()a-z0-9._]*[0-9][a-z0-9._]*)$"); // rel contains at least one digit + private static final Pattern NAME_VER_REL_PATTERN = + Pattern.compile("^(.+?)?" + // name + "-([+()a-z0-9._]*[0-9]+\\.[a-z0-9._]*)" + // ver: contains at least one digit and dot + "(?:-([+()a-z0-9._]*[0-9][a-z0-9._]*))?$"); // rel: contains at least one digit private static final Pattern NAME_PATTERN = Pattern.compile("^[+()a-zA-Z0-9._-]+$"); private final Optional<String> epoch; @@ -74,15 +74,6 @@ public class YumPackageName { architecture = packageName.architecture; } - /** - * Set the epoch of the YUM package. - * - * <p>WARNING: Should only be invoked if the YUM package actually has an epoch. Typically - * YUM packages doesn't have one explicitly set, and in case "0" will be used with - * {@link #toVersionLockName(Version)} (otherwise it fails), but it will be absent from an - * install with {@link #toName(Version)} (otherwise it fails). This typically means that - * you should set this only if the epoch is != "0".</p> - */ public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; } public Builder setName(String name) { this.name = name; return this; } public Builder setVersion(String version) { this.version = Optional.of(version); return this; } @@ -100,7 +91,6 @@ public class YumPackageName { Optional<String> architecture) { if (Strings.isNullOrEmpty(name)) throw new IllegalArgumentException("name cannot be null or empty"); - this.epoch = epoch; this.name = name; this.version = version; @@ -116,6 +106,7 @@ public class YumPackageName { * <ol> * <li>name * <li>name.arch + * <li>name-ver * <li>name-ver-rel * <li>name-ver-rel.arch * <li>name-epoch:ver-rel.arch @@ -125,19 +116,12 @@ public class YumPackageName { * @throws IllegalArgumentException if spec does not specify a package name. * @see #parseString(String) */ - public static YumPackageName fromString(final String packageSpec) { + public static YumPackageName fromString(String packageSpec) { String spec = packageSpec; Optional<String> epoch = Optional.empty(); String name = null; - // packageSpec spec - // name name - // name.arch name.arch - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel.arch - // name-epoch:ver-rel.arch name-epoch:ver-rel.arch - // epoch:name-ver-rel.arch epoch:name-ver-rel.arch - + // Parse epoch and remove it from spec int epochColon = spec.indexOf(':'); if (epochColon >= 0) { Matcher epochMatcher = EPOCH_PATTERN.matcher(spec.substring(0, epochColon)); @@ -148,17 +132,12 @@ public class YumPackageName { name = epochMatcher.group(2); epoch = Optional.of(epochMatcher.group(3)); - spec = spec.substring(epochColon + 1); + spec = epochColon == 0 + ? spec.substring(epochColon + 1) + : spec.substring(0, epochColon - 1) + spec.substring(epochColon + 1); } - // packageSpec spec - // name name - // name.arch name.arch - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel.arch - // name-epoch:ver-rel.arch ver-rel.arch (non-null name) - // epoch:name-ver-rel.arch name-ver-rel.arch - + // Parse architecture and remove it from spec Optional<String> architecture = Optional.empty(); Matcher architectureMatcher = ARCHITECTURE_PATTERN.matcher(spec); if (architectureMatcher.find()) { @@ -166,44 +145,27 @@ public class YumPackageName { spec = spec.substring(0, architectureMatcher.start()); } - // packageSpec spec - // name name - // name.arch name - // name-ver-rel name-ver-rel - // name-ver-rel.arch name-ver-rel - // name-epoch:ver-rel.arch ver-rel (non-null name) - // epoch:name-ver-rel.arch name-ver-rel - + // Parse name, version and release and remove the latter two from spec Optional<String> version = Optional.empty(); Optional<String> release = Optional.empty(); Matcher matcher = NAME_VER_REL_PATTERN.matcher(spec); if (matcher.find()) { - // spec format one of: - // 1. name-ver-rel - // 2. ver-rel - - spec = matcher.group(2); + spec = matcher.group(1); if (spec == null) { if (name == null) { throw new IllegalArgumentException("No package name was found: " + packageSpec); } spec = name; // makes spec hold the package name in all cases below. - } else if (name != null) { - throw new IllegalArgumentException("Ambiguous package names were found for " + - packageSpec + ": '" + name + "' and '" + spec + "'"); } - version = Optional.of(matcher.group(3)); - release = Optional.of(matcher.group(4)); + version = Optional.of(matcher.group(2)); + release = Optional.ofNullable(matcher.group(3)); } - // packageSpec spec - // name name - // name.arch name - // name-ver-rel name - // name-ver-rel.arch name - // name-epoch:ver-rel.arch name - // epoch:name-ver-rel.arch name + // Set default epoch if we have a version + if (version.isPresent() && epoch.isEmpty()) { + epoch = Optional.of("0"); + } if (!NAME_PATTERN.matcher(spec).find()) { throw new IllegalArgumentException("Bad package name in " + packageSpec + ": '" + spec + "'"); @@ -229,13 +191,12 @@ public class YumPackageName { public Optional<String> getArchitecture() { return architecture; } /** Return package name, omitting components that are not specified. */ - public String toName(Version yumVersion) { + public String toName() { StringBuilder builder = new StringBuilder(); - boolean isBare = version.isEmpty() && release.isEmpty() && architecture.isEmpty(); char nextDelimiter; builder.append(name); // Fully versioned package names must always include epoch in Yum 4 - epoch.or(() -> Optional.of("0").filter(v -> !isBare)) + epoch.or(() -> Optional.of("0").filter(v -> version.isPresent())) .ifPresent(ep -> builder.append('-').append(ep)); nextDelimiter = ':'; version.ifPresent(s -> builder.append(nextDelimiter).append(s)); @@ -249,7 +210,7 @@ public class YumPackageName { * * @throws IllegalStateException if any field required for the version lock spec is missing */ - public String toVersionLockName(Version yumVersion) { + public String toVersionLockName() { Builder b = new Builder(this).setArchitecture("*"); if (epoch.isEmpty()) { b.setEpoch("0"); @@ -257,7 +218,7 @@ public class YumPackageName { YumPackageName lockSpec = b.build(); if (lockSpec.getVersion().isEmpty()) throw new IllegalStateException("Version is missing for YUM package " + name); if (lockSpec.getRelease().isEmpty()) throw new IllegalStateException("Release is missing for YUM package " + name); - return lockSpec.toName(yumVersion); + return lockSpec.toName(); } public boolean isSubsetOf(YumPackageName other) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java index a73f5d04704..e48a0fa2683 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; @@ -17,20 +16,10 @@ import java.util.stream.Stream; public class YumTester extends Yum { private final TestTerminal terminal; - private final Version yumVersion; public YumTester(TestTerminal terminal) { - this(terminal, YumVersion.rhel8); - } - - public YumTester(TestTerminal terminal, YumVersion yumVersion) { super(terminal); this.terminal = terminal; - this.yumVersion = yumVersion.asVersion(); - } - - public Version yumVersion() { - return yumVersion; } public GenericYumCommandExpectation expectInstall(String... packages) { @@ -81,10 +70,6 @@ public class YumTester extends Yum { } } - protected void expectYumVersion() { - terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n"); - } - private YumTester execute(String output) { if (commandType == CommandType.install) terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(1, "Not installed")); @@ -106,7 +91,7 @@ public class YumTester extends Yum { if (commandType == CommandType.upgrade && packages.size() > 1) cmd.append(" --setopt skip_missing_names_on_update=False"); packages.forEach(pkg -> { - String name = pkg.toName(yumVersion); + String name = pkg.toName(); if (name.contains("(") || name.contains(")")) { // Ugly hack to handle implicit quoting done in com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine name = "\"" + name + "\""; } @@ -114,26 +99,20 @@ public class YumTester extends Yum { }); cmd.append(" 2>&1"); - expectYumVersion(); terminal.expectCommand(cmd.toString(), 0, output); return YumTester.this; } } public class InstallFixedCommandExpectation extends GenericYumCommandExpectation { + private InstallFixedCommandExpectation(String yumPackage) { super(CommandType.installFixed, yumPackage); } @Override - protected void expectYumVersion() {} - - @Override public YumTester andReturn(boolean value) { - // Pretend package is already correctly version-locked to simplify expectations - terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n"); - - terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName(yumVersion)); + terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName()); return super.andReturn(value); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java deleted file mode 100644 index 8eba4561805..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.yum; - -import com.yahoo.component.Version; - -/** - * Red Hat versions and their associated Yum/DNF major version. - * - * @author mpolden - */ -public enum YumVersion { - - rhel8(4); - - private final Version version; - - YumVersion(int yumMajor) { - this.version = new Version(yumMajor, 0, 0); - } - - public Version asVersion() { - return version; - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java index 0341558d538..a4dc3ac359a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.yum; -import com.yahoo.component.Version; import org.junit.Test; import java.util.Optional; @@ -26,7 +25,7 @@ public class YumPackageNameTest { .setRelease("71.git3e8e77d.el7.centos.1") .setArchitecture("x86_64") .build(); - assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName(Version.fromString("4"))); + assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName()); } @Test @@ -61,11 +60,21 @@ public class YumPackageNameTest { null, null, "x86_64", - "docker-engine-selinux-0.x86_64", + "docker-engine-selinux.x86_64", null); + // name-ver + verifyPackageName("docker-engine-selinux-1.12.6", + "0", + "docker-engine-selinux", + "1.12.6", + null, + null, + "docker-engine-selinux-0:1.12.6", + null); + // name-ver-rel - verifyPackageName("docker-engine-selinux-0:1.12.6-1.el7", + verifyPackageName("docker-engine-selinux-1.12.6-1.el7", "0", "docker-engine-selinux", "1.12.6", @@ -75,7 +84,7 @@ public class YumPackageNameTest { "docker-engine-selinux-0:1.12.6-1.el7.*"); // name-ver-rel.arch - verifyPackageName("docker-engine-selinux-0:1.12.6-1.el7.x86_64", + verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64", "0", "docker-engine-selinux", "1.12.6", @@ -97,7 +106,7 @@ public class YumPackageNameTest { // epoch:name-ver-rel.arch verifyPackageName( - "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", + "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64", "2", "docker", "1.12.6", @@ -107,7 +116,7 @@ public class YumPackageNameTest { "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*"); } - private void verifyPackageName(String packageName, + private void verifyPackageName(String input, String epoch, String name, String version, @@ -115,32 +124,33 @@ public class YumPackageNameTest { String architecture, String toName, String toVersionName) { - YumVersion yumVersion = YumVersion.rhel8; - YumPackageName yumPackageName = YumPackageName.fromString(packageName); - verifyValue(epoch, yumPackageName.getEpoch()); - verifyValue(name, Optional.of(yumPackageName.getName())); - verifyValue(version, yumPackageName.getVersion()); - verifyValue(release, yumPackageName.getRelease()); - verifyValue(architecture, yumPackageName.getArchitecture()); - verifyValue(toName, Optional.of(yumPackageName.toName(yumVersion.asVersion()))); + YumPackageName yumPackageName = YumPackageName.fromString(input); + assertPackageField("epoch", epoch, yumPackageName.getEpoch()); + assertPackageField("name", name, Optional.of(yumPackageName.getName())); + assertPackageField("version", version, yumPackageName.getVersion()); + assertPackageField("release", release, yumPackageName.getRelease()); + assertPackageField("architecture", architecture, yumPackageName.getArchitecture()); + assertPackageField("toName()", toName, Optional.of(yumPackageName.toName())); if (toVersionName == null) { try { - yumPackageName.toVersionLockName(yumVersion.asVersion()); + yumPackageName.toVersionLockName(); fail(); } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsStringIgnoringCase("Version is missing ")); + assertTrue("Exception message contains expected substring: " + e.getMessage(), + e.getMessage().contains("Version is missing ") || + e.getMessage().contains("Release is missing ")); } } else { - assertEquals(toVersionName, yumPackageName.toVersionLockName(yumVersion.asVersion())); + assertEquals(toVersionName, yumPackageName.toVersionLockName()); } } - private void verifyValue(String value, Optional<String> actual) { - if (value == null) { - assertFalse(actual.isPresent()); + private void assertPackageField(String field, String expected, Optional<String> actual) { + if (expected == null) { + assertFalse(field + " is not present", actual.isPresent()); } else { - assertEquals(value, actual.get()); + assertEquals(field + " has expected value", expected, actual.get()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java index 2699135b5b9..62fd7410e95 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -79,20 +79,6 @@ public class YumTest { @Test public void testAlreadyInstalled() { mockRpmQuery("package-1", null); - mockYumVersion(); - terminal.expectCommand( - "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", - 0, - "foobar\nNothing to do\n"); - - assertFalse(yum - .install("package-1", "package-2") - .enableRepo("repo1", "repo2") - .converge(taskContext)); - - // RHEL 8 - mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -104,18 +90,6 @@ public class YumTest { @Test public void testAlreadyUpgraded() { - mockYumVersion(); - terminal.expectCommand( - "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", - 0, - "foobar\nNo packages marked for update\n"); - - assertFalse(yum - .upgrade("package-1", "package-2") - .converge(taskContext)); - - // RHEL 8 - mockYumVersion(); terminal.expectCommand( "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1", 0, @@ -128,19 +102,6 @@ public class YumTest { @Test public void testAlreadyRemoved() { mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockYumVersion(); - terminal.expectCommand( - "yum remove --assumeyes package-1 package-2 2>&1", - 0, - "foobar\nNo Packages marked for removal\n"); - - assertFalse(yum - .remove("package-1", "package-2") - .converge(taskContext)); - - // RHEL 8 - mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1")); - mockYumVersion(); terminal.expectCommand( "yum remove --assumeyes package-1 package-2 2>&1", 0, @@ -160,7 +121,6 @@ public class YumTest { @Test public void testInstall() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -181,7 +141,6 @@ public class YumTest { @Test public void testInstallWithEnablerepo() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 0, @@ -195,23 +154,6 @@ public class YumTest { @Test public void testWithVersionLock() { - mockYumVersion(); - terminal.expectCommand("yum versionlock list 2>&1", - 0, - "Repository chef_rpms-release is listed more than once in the configuration\n" + - "0:chef-12.21.1-1.el7.*\n"); - terminal.expectCommand("yum versionlock add --assumeyes \"package-0:0.10-654.el7.*\" 2>&1"); - terminal.expectCommand( - "yum install --assumeyes package-0:0.10-654.el7.x86_64 2>&1", - 0, - "installing"); - - assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7.x86_64")).converge(taskContext)); - } - - @Test - public void testWithVersionLockYum4() { - mockYumVersion(); terminal.expectCommand("yum versionlock list 2>&1", 0, "Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n"); @@ -232,7 +174,6 @@ public class YumTest { @Test public void testWithDifferentVersionLock() { - mockYumVersion(); terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + @@ -257,7 +198,6 @@ public class YumTest { @Test public void testWithExistingVersionLock() { - mockYumVersion(); terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + @@ -273,7 +213,6 @@ public class YumTest { @Test public void testWithDowngrade() { - mockYumVersion(); terminal.expectCommand("yum versionlock list 2>&1", 0, "Repository chef_rpms-release is listed more than once in the configuration\n" + @@ -294,7 +233,6 @@ public class YumTest { @Test(expected = ChildProcessFailureException.class) public void testFailedInstall() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1", 1, @@ -310,7 +248,6 @@ public class YumTest { @Test public void testUnknownPackages() { mockRpmQuery("package-1", null); - mockYumVersion(); terminal.expectCommand( "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1", 0, @@ -337,15 +274,10 @@ public class YumTest { @Test public void allowToCallUpgradeWithNoPackages() { - mockYumVersion(); terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); yum.upgrade().converge(taskContext); } - private void mockYumVersion() { - terminal.expectCommand("yum --version 2>&1", 0, YumVersion.rhel8.asVersion().toFullString() + "\ntrailing garbage\n"); - } - private void mockRpmQuery(String packageName, YumPackageName installedOrNull) { new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java index bb4bb2686a3..93fffffb2fa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java @@ -37,7 +37,7 @@ public class YumTesterTest { assertYumMethod(yum -> yum.expectRemove(packages).withEnableRepo(repos), yum -> yum.remove(List.of(packages)).enableRepo(repos).converge(context)); - assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName(yum.yumVersion())).withEnableRepo(repos), + assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withEnableRepo(repos), yum -> yum.installFixedVersion(minimalPackage).enableRepo(repos).converge(context)); } diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 30aa76658fd..e7fdc560bc5 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -26,6 +26,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -79,10 +85,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> <version>${project.version}</version> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index b465917e9c9..b799e0056f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -107,8 +107,13 @@ public class AllocatableClusterResources { } public boolean preferableTo(AllocatableClusterResources other) { - if (this.fulfilment < 1 || other.fulfilment < 1) - return this.fulfilment > other.fulfilment; // we always want to fulfil as much as possible + if (this.fulfilment < 1 || other.fulfilment < 1) // always fulfil as much as possible + return this.fulfilment > other.fulfilment; + + if (clusterSpec.type().isContent() // always prefer local storage on content nodes + && this.realResources.storageType() != other.realResources().nodeResources().storageType()) + return this.realResources.storageType() == NodeResources.StorageType.local; + return this.cost() < other.cost(); // otherwise, prefer lower cost } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 59556418fb5..2b64cc86c9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Locale; @@ -40,7 +41,10 @@ public class NodeResourceLimits { /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ public boolean isWithinRealLimits(NodeCandidate candidateNode, ClusterSpec cluster) { if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes - return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, cluster.isExclusive()), + // This node is allocated exclusively if that has been explicitly requested, or if the host of the node was + // provisioned exclusively + boolean exclusive = cluster.isExclusive() || candidateNode.parent.flatMap(Node::exclusiveTo).isPresent(); + return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, exclusive), cluster.type()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 761afed23cc..5c452989f56 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; @@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; -import org.junit.Ignore; import org.junit.Test; import java.time.Instant; @@ -150,7 +148,7 @@ public class DynamicAllocationTest { } @Test - public void test_allocation_balancing() { + public void allocation_balancing() { // Here we test balancing between cpu and memory and ignore disk ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); @@ -234,14 +232,12 @@ public class DynamicAllocationTest { fail("Two groups have been allocated to the same parent host"); } - @Ignore // TODO: Re-enable if we reintroduce spare capacity requirement @Test public void spare_capacity_used_only_when_replacement() { - // Use spare capacity only when replacement (i.e one node is failed) - // Test should allocate as much capacity as possible, verify that it is not possible to allocate one more unit - // Verify that there is still capacity (available spare) - // Fail one node and redeploy, Verify that one less node is empty. - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavorsConfig(flavorsConfig()) + .spareCount(2) + .build(); // Setup test ApplicationId application1 = ProvisioningTester.applicationId(); @@ -252,7 +248,7 @@ public class DynamicAllocationTest { // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1"); List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); assertEquals(2, initialSpareCapacity.size()); @@ -264,10 +260,9 @@ public class DynamicAllocationTest { tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> finalSpareCapacity = findSpareCapacity(tester); - assertEquals(1, finalSpareCapacity.size()); } @@ -278,7 +273,7 @@ public class DynamicAllocationTest { tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); assertEquals(0, initialSpareCapacity.size()); @@ -291,7 +286,7 @@ public class DynamicAllocationTest { tester.activateTenantHosts(); ApplicationId application1 = ProvisioningTester.applicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); - tester.activate(application1, ImmutableSet.copyOf(hosts)); + tester.activate(application1, Set.copyOf(hosts)); } @Test(expected = OutOfCapacityException.class) @@ -315,8 +310,8 @@ public class DynamicAllocationTest { tester.activate(application, hosts); NodeList activeNodes = tester.nodeRepository().nodes().list().owner(application); - assertEquals(ImmutableSet.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary()); - assertEquals(ImmutableSet.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary()); + assertEquals(Set.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary()); + assertEquals(Set.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary()); } @Test @@ -374,7 +369,7 @@ public class DynamicAllocationTest { } @Test - public void nodeResourcesAreRelaxedInDev() { + public void node_resources_are_relaxed_in_dev() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -394,7 +389,7 @@ public class DynamicAllocationTest { } @Test - public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { + public void switching_from_legacy_flavor_syntax_to_resources_does_not_cause_reallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); tester.activateTenantHosts(); @@ -413,7 +408,7 @@ public class DynamicAllocationTest { } @Test - public void testPreferExclusiveNetworkSwitch() { + public void prefer_exclusive_network_switch() { // Hosts are provisioned, without switch information ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); NodeResources hostResources = new NodeResources(32, 128, 2000, 10); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 6f1e2630434..0b7b9f2fa13 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -364,7 +364,7 @@ public class DynamicProvisioningTest { } @Test - public void test_any_disk_prefers_remote() { + public void test_any_disk_prefers_remote_for_container() { int memoryTax = 3; int localDiskTax = 55; // Disk tax is not included in flavor resources but memory tax is @@ -383,7 +383,7 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), resources(6, 3, 3, 25, 400, fast, StorageType.any))); @@ -392,6 +392,35 @@ public class DynamicProvisioningTest { app1, cluster1); } + @Test + public void test_any_disk_prefers_local_for_content() { + int memoryTax = 3; + int localDiskTax = 55; + // Disk tax is not included in flavor resources but memory tax is + List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, local)), + new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, local)), + new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), + new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote))); + + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + .flavors(flavors) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) + .nameResolver(nameResolver) + .resourcesCalculator(memoryTax, localDiskTax) + .build(); + + tester.activateTenantHosts(); + + ApplicationId app1 = ProvisioningTester.applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), + resources(6, 3, 3, 25, 400, fast, StorageType.any))); + tester.assertNodes("'any' selects a flavor with local storage", + 6, 2, 2, 20, 200, fast, local, + app1, cluster1); + } + private void prepareAndActivate(ApplicationId application, ClusterSpec clusterSpec, int nodes, int groups, NodeResources resources) { List<HostSpec> prepared = tester.prepare(application, clusterSpec, nodes, groups, resources); NodeList provisionedHosts = tester.nodeRepository().nodes().list(Node.State.provisioned).nodeType(NodeType.host); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index daafc81e0f3..8902bf1641c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.orchestrator.status; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; import com.yahoo.path.Path; @@ -54,16 +53,10 @@ public class ZkStatusService implements StatusService { private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>(); @Inject - public ZkStatusService( - @Component Curator curator, - @Component Metric metric, - @Component Timer timer, - @Component AntiServiceMonitor antiServiceMonitor) { - this(curator, - metric, - timer, - new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)), - antiServiceMonitor); + public ZkStatusService(Curator curator, Metric metric, Timer timer, AntiServiceMonitor antiServiceMonitor) { + this(curator, metric, timer, + new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)), + antiServiceMonitor); } /** Non-private for testing only. */ diff --git a/parent/pom.xml b/parent/pom.xml index 908491835ea..2a8527f80ea 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -863,6 +863,7 @@ --> <curator.version>5.1.0</curator.version> <jna.version>4.5.2</jna.version> + <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> <junit.version>5.7.0</junit.version> <maven-assembly-plugin.version>3.1.1</maven-assembly-plugin.version> diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 55b3b66a75c..dfcc7542e09 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -143,6 +143,7 @@ vespa_define_module( src/tests/proton/reprocessing/reprocessing_runner src/tests/proton/server src/tests/proton/server/disk_mem_usage_filter + src/tests/proton/server/disk_mem_usage_metrics src/tests/proton/server/disk_mem_usage_sampler src/tests/proton/server/health_adapter src/tests/proton/server/memory_flush_config_updater diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt new file mode 100644 index 00000000000..fbd557991c8 --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_disk_mem_usage_metrics_test_app TEST + SOURCES + disk_mem_usage_metrics_test.cpp + DEPENDS + searchcore_server + GTest::GTest +) +vespa_add_test(NAME searchcore_disk_mem_usage_metrics_test_app COMMAND searchcore_disk_mem_usage_metrics_test_app) diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp new file mode 100644 index 00000000000..569633c518b --- /dev/null +++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/server/disk_mem_usage_metrics.h> +#include <vespa/searchcore/proton/server/disk_mem_usage_state.h> +#include <vespa/vespalib/gtest/gtest.h> + + +using proton::DiskMemUsageMetrics; +using proton::DiskMemUsageState; +using proton::ResourceUsageState; + +bool +expect_metrics(double disk_usage, double disk_utilization, double memory_usage, double memory_utilization, const DiskMemUsageMetrics &dm_metrics) +{ + bool result = true; + EXPECT_DOUBLE_EQ(disk_usage, dm_metrics.get_disk_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(disk_utilization, dm_metrics.get_disk_utilization()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_usage, dm_metrics.get_memory_usage()) << (result = false, ""); + EXPECT_DOUBLE_EQ(memory_utilization, dm_metrics.get_memory_utilization()) << (result = false, ""); + return result; +} + +TEST(DiskMemUsageMetricsTest, default_value_is_zero) +{ + DiskMemUsageMetrics dm_metrics; + EXPECT_TRUE(expect_metrics(0.0, 0.0, 0.0, 0.0, dm_metrics)); +} + +TEST(DiskMemUsageMetricsTest, merging_uses_max) +{ + DiskMemUsageMetrics dm_metrics({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); + EXPECT_TRUE(expect_metrics(0.4, 0.8, 0.3, 0.6, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.4, 0.4), ResourceUsageState(0.5, 0.4)}); + EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); + dm_metrics.merge({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)}); + EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index c0b5bcab791..b8d1d2f17e6 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -71,6 +71,7 @@ DocsumContext::createReply() reply->docsums.resize(_docsumState._docsumcnt); SymbolTable::UP symbols = std::make_unique<SymbolTable>(); IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId()); + _docsumState._omit_summary_features = rci.outputClass->omit_summary_features(); for (uint32_t i = 0; i < _docsumState._docsumcnt; ++i) { buf.reset(); uint32_t docId = _docsumState._docsumbuf[i]; @@ -114,6 +115,7 @@ DocsumContext::createSlimeReply() const Symbol docsumSym = response->insert(DOCSUM); IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId()); + _docsumState._omit_summary_features = rci.outputClass->omit_summary_features(); uint32_t i(0); for (i = 0; (i < _docsumState._docsumcnt) && !_request.expired(); ++i) { uint32_t docId = _docsumState._docsumbuf[i]; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index d2ebb7a58a2..285e259e1a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchcore_server STATIC ddbstate.cpp disk_mem_usage_filter.cpp disk_mem_usage_forwarder.cpp + disk_mem_usage_metrics.cpp disk_mem_usage_sampler.cpp docstorevalidator.cpp document_db_config_owner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index ef77081132b..cbd378da9a0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -154,13 +154,14 @@ DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), _hwInfo(hwInfo), + _acceptWrite(true), _memoryStats(), _diskUsedSizeBytes(), _transient_memory_usage(0u), _config(), _state(), - _acceptWrite(true), _dmstate(), + _disk_mem_usage_metrics(), _listeners() { } @@ -254,6 +255,15 @@ DiskMemUsageFilter::usageState() const return _dmstate; } +DiskMemUsageMetrics +DiskMemUsageFilter::get_metrics() const +{ + Guard guard(_lock); + DiskMemUsageMetrics result(_disk_mem_usage_metrics); + _disk_mem_usage_metrics = DiskMemUsageMetrics(_dmstate); + return result; +} + bool DiskMemUsageFilter::acceptWriteOperation() const { @@ -295,6 +305,7 @@ DiskMemUsageFilter::notifyDiskMemUsage(const Guard &guard, DiskMemUsageState sta return; } _dmstate = state; + _disk_mem_usage_metrics.merge(state); for (const auto &listener : _listeners) { listener->notifyDiskMemUsage(_dmstate); } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index cdb9fc5f4cb..5bec34e51d9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -4,6 +4,7 @@ #include "i_disk_mem_usage_notifier.h" #include "disk_mem_usage_state.h" +#include "disk_mem_usage_metrics.h" #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> @@ -40,16 +41,18 @@ public: }; private: - mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state + mutable Mutex _lock; HwInfo _hwInfo; + std::atomic<bool> _acceptWrite; + // Following member variables are protected by _lock vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; size_t _transient_memory_usage; size_t _transient_disk_usage; Config _config; State _state; - std::atomic<bool> _acceptWrite; DiskMemUsageState _dmstate; + mutable DiskMemUsageMetrics _disk_mem_usage_metrics; std::vector<IDiskMemUsageListener *> _listeners; void recalcState(const Guard &guard); // called with _lock held @@ -73,6 +76,7 @@ public: Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; + DiskMemUsageMetrics get_metrics() const; bool acceptWriteOperation() const override; State getAcceptState() const override; void addDiskMemUsageListener(IDiskMemUsageListener *listener) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp new file mode 100644 index 00000000000..2a97c56e63f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "disk_mem_usage_metrics.h" +#include "disk_mem_usage_state.h" +#include <algorithm> + +namespace proton { + +DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept + : DiskMemUsageMetrics(DiskMemUsageState()) +{ +} + +DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept + : _disk_usage(usage_state.diskState().usage()), + _disk_utilization(usage_state.diskState().utilization()), + _memory_usage(usage_state.memoryState().usage()), + _memory_utilization(usage_state.memoryState().utilization()) +{ +} + +void +DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept +{ + _disk_usage = std::max(_disk_usage, usage_state.diskState().usage()); + _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); + _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); + _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h new file mode 100644 index 00000000000..34d3bb36b40 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace proton { + +class DiskMemUsageState; + +/** + * Class containing disk and memory usage in a form suitable for + * metrics reporting. + */ +class DiskMemUsageMetrics +{ + double _disk_usage; + double _disk_utilization; + double _memory_usage; + double _memory_utilization; + +public: + DiskMemUsageMetrics() noexcept; + DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept; + void merge(const DiskMemUsageState &usage_state) noexcept; + double get_disk_usage() const noexcept { return _disk_usage; } + double get_disk_utilization() const noexcept { return _disk_utilization; } + double get_memory_usage() const noexcept { return _memory_usage; } + double get_memory_utilization() const noexcept { return _memory_utilization; } +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index adae323c1d9..ef72a52eb02 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -756,11 +756,11 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) } const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter(); - DiskMemUsageState usageState = usageFilter.usageState(); - metrics.resourceUsage.disk.set(usageState.diskState().usage()); - metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization()); - metrics.resourceUsage.memory.set(usageState.memoryState().usage()); - metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization()); + auto dm_metrics = usageFilter.get_metrics(); + metrics.resourceUsage.disk.set(dm_metrics.get_disk_usage()); + metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization()); + metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); + metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h index 29f9959aea9..0b6f34ba055 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h @@ -18,7 +18,7 @@ private: public: ResourceUsageState() - : _limit(0), + : _limit(1.0), _usage(0) { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 99a372d9676..6f042ee3907 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -36,6 +36,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _parsedLocations(), _summaryFeatures(nullptr), _summaryFeaturesCached(false), + _omit_summary_features(false), _rankFeatures(nullptr), _matching_elements(), _jsonStringer() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index db0f8e6e8ad..88a95d0446c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -84,6 +84,7 @@ public: // used by SummaryFeaturesDFW FeatureSet::SP _summaryFeatures; bool _summaryFeaturesCached; + bool _omit_summary_features; // used by RankFeaturesDFW FeatureSet::SP _rankFeatures; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 8066a5e65db..f52cc62af92 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -15,7 +15,8 @@ ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & field _nameMap(), _fieldEnum(fieldEnum), _enumMap(), - _dynInfo(NULL) + _dynInfo(NULL), + _omit_summary_features(false) { } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h index 52e331cd365..88c3552387b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h @@ -141,6 +141,9 @@ private: util::StringEnum &_fieldEnum; // fieldname -> f.n. enum value [SHARED] std::vector<int> _enumMap; // fieldname enum value -> entry index DynamicInfo *_dynInfo; // fields overridden and generated + // Whether or not summary features should be omitted when filling this summary class. + // As default, summary features are always included. + bool _omit_summary_features; public: typedef std::unique_ptr<ResultClass> UP; @@ -278,6 +281,14 @@ public: { return (offset < _entries.size()) ? &_entries[offset] : NULL; } + + void set_omit_summary_features(bool value) { + _omit_summary_features = value; + } + + bool omit_summary_features() const { + return _omit_summary_features; + } }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index ead3a4a2f9d..579779efb73 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -127,25 +127,27 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const int maxclassID = 0x7fffffff; // avoid negative classids _defaultSummaryId = cfg.defaultsummaryid; for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) { - if (cfg.classes[i].name.empty()) { + const auto& cfg_class = cfg.classes[i]; + if (cfg_class.name.empty()) { LOG(warning, "%s classes[%d]: empty name", configId, i); } - int classID = cfg.classes[i].id; + int classID = cfg_class.id; if (classID < 0 || classID > maxclassID) { LOG(error, "%s classes[%d]: bad id %d", configId, i, classID); rc = false; break; } - ResultClass *resClass = AddResultClass(cfg.classes[i].name.c_str(), classID); + ResultClass *resClass = AddResultClass(cfg_class.name.c_str(), classID); if (resClass == nullptr) { - LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg.classes[i].name.c_str()); + LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg_class.name.c_str()); rc = false; break; } - for (unsigned int j = 0; rc && (j < cfg.classes[i].fields.size()); j++) { - const char *fieldtype = cfg.classes[i].fields[j].type.c_str(); - const char *fieldname = cfg.classes[i].fields[j].name.c_str(); - LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg.classes[i].name.c_str(), fieldname, fieldtype); + resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures); + for (unsigned int j = 0; rc && (j < cfg_class.fields.size()); j++) { + const char *fieldtype = cfg_class.fields[j].type.c_str(); + const char *fieldname = cfg_class.fields[j].name.c_str(); + LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg_class.name.c_str(), fieldname, fieldtype); if (strcmp(fieldtype, "integer") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_INT); } else if (strcmp(fieldtype, "short") == 0) { @@ -179,12 +181,12 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const } else if (strcmp(fieldtype, "featuredata") == 0) { rc = resClass->AddConfigEntry(fieldname, RES_FEATUREDATA); } else { - LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg.classes[i].name.c_str(), j, fieldtype); + LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg_class.name.c_str(), j, fieldtype); rc = false; break; } if (!rc) { - LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg.classes[i].name.c_str(), j, fieldname); + LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg_class.name.c_str(), j, fieldname); break; } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 425faff6a67..4d81f4505a7 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -30,6 +30,9 @@ static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { + if (state->_omit_summary_features) { + return; + } if ( ! state->_summaryFeatures) { state->_callback.FillSummaryFeatures(state, _env); if ( !state->_summaryFeatures) { // still no summary features to write diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 578fcc83006..df28737f352 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -16,14 +16,6 @@ <description>Service monitor component for hosted vespa.</description> <dependencies> - <!-- compile scope --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.5</version> - <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new --> - <scope>compile</scope> - </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> @@ -34,6 +26,12 @@ <!-- provided scope --> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 11d0406fcc5..4aa66e9b1a4 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -90,6 +90,8 @@ config-model-api-jar-with-dependencies.jar, config-model-jar-with-dependencies.jar, container-disc-jar-with-dependencies.jar, + model-evaluation-jar-with-dependencies.jar, + model-integration-jar-with-dependencies.jar, vespajlib.jar </discPreInstallBundle> </configuration> diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index 3f3937658d2..b30a7ad4cd0 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -167,6 +167,7 @@ StartCommand() { --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ --add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.library.path="$VESPA_HOME/lib64" \ diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 90ab2a81e0c..1b03891f313 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -25,6 +25,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -131,14 +137,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> <exclusions> @@ -154,6 +152,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-annotations</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -170,7 +172,7 @@ <!-- required by java-jwt --> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> - <version>1.15</version> + <version>${commons.codec.version}</version> <scope>compile</scope> </dependency> </dependencies> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java index 97bc7dc838e..f80c28424f2 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java @@ -35,7 +35,7 @@ public class VespaOutputFormat extends OutputFormat { public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { VespaCounters counters = VespaCounters.get(context); VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride); - return configuration.useLegacyClient() + return configuration.useLegacyClient().orElse(true) ? new LegacyVespaRecordWriter(configuration, counters) : new VespaRecordWriter(configuration, counters); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 7219e621486..1be794f8e11 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -6,6 +6,7 @@ import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.io.StringReader; +import java.util.Optional; import java.util.Properties; public class VespaConfiguration { @@ -131,7 +132,11 @@ public class VespaConfiguration { return getInt(PROGRESS_REPORT, 1000); } - public boolean useLegacyClient() { return getBoolean(USE_LEGACY_CLIENT, true); } + public Optional<Boolean> useLegacyClient() { + String raw = getString(USE_LEGACY_CLIENT); + if (raw == null || raw.trim().isEmpty()) return Optional.empty(); + return Optional.of(Boolean.parseBoolean(raw)); + } public String getString(String name) { if (override != null && override.containsKey(name)) { @@ -191,6 +196,7 @@ public class VespaConfiguration { sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n"); sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n"); sb.append(NUM_RETRIES + ": " + numRetries() +"\n"); + sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n"); return sb.toString(); } diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java index 259ae2602c4..69c0e343872 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java @@ -39,6 +39,8 @@ public class GenerateTestDescriptorMojo extends AbstractMojo { } private void analyzeTestClasses(TestAnnotationAnalyzer analyzer) throws MojoExecutionException { + if (! Files.exists(testClassesDirectory())) return; + try (Stream<Path> files = Files.walk(testClassesDirectory())) { files .filter(f -> f.toString().endsWith(".class")) diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 834c3d7c988..17443b11b6a 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -38,11 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-http-client</artifactId> <version>${project.version}</version> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 9db296e33cd..770aa9bdb81 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1133,7 +1133,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { default: response.writeMessage(error.get() != null ? error.get() : message != null ? message : "Visiting failed"); if (getVisitorStatistics() != null) - response.writeDocumentCount(getVisitorStatistics().getDocumentsReturned()); + response.writeDocumentCount(getVisitorStatistics().getDocumentsVisited()); response.respond(Response.Status.BAD_GATEWAY); } diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 25cc678fe66..7140c490460 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -50,6 +50,12 @@ <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -57,19 +63,27 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <!-- Needed to have the same version as slf4j-api --> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> - </dependency> - <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.2 due to @@ -79,6 +93,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml index 2f0029169ea..e40ba8fa161 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml @@ -33,15 +33,24 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.6.2</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.2 due to @@ -51,6 +60,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml index a4568d3585f..3074f3d2117 100644 --- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.3/pom.xml @@ -33,15 +33,24 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.6.3</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>${slf4j.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <!-- snappy-java and metrics-core are included here to be able to work with ZooKeeper 3.6.3 due to @@ -51,6 +60,12 @@ <artifactId>metrics-core</artifactId> <scope>compile</scope> <version>3.2.5</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.xerial.snappy</groupId> |