diff options
| author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-20 09:42:19 +0100 |
|---|---|---|
| committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-02-20 09:42:19 +0100 |
| commit | 5acf4c47e98674cdf73289a782dfda9da7041ead (patch) | |
| tree | 9a2720a3326326cc2a0b69d29b6877e4039d5f18 | |
| parent | d2449a3e66075e7d680263a204302e83b5ba0148 (diff) | |
| parent | 1cc70ca6f328e7e88e8b4e279cac7544624f055b (diff) | |
Merge branch 'master' into bratseth/node-metrics
1352 files changed, 27343 insertions, 14063 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a4ef1bdba25..8aa853e8c39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,9 @@ vespa_use_default_cxx_compiler() vespa_use_default_java_home() project(vespa CXX C) +vespa_use_default_vespa_unprivileged() +vespa_use_default_cmake_install_prefix() +vespa_use_default_vespa_user() vespa_use_default_build_settings() # allows import of project in CLion on OSX @@ -1 +1 @@ -7.0.0 +7.164.0 diff --git a/application/pom.xml b/application/pom.xml index 7bcda0ce386..d173fec19cf 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -119,8 +119,8 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>test</scope> </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index 5f9b1f51863..b200244a21c 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -2,6 +2,7 @@ package com.yahoo.application; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; @@ -117,11 +118,13 @@ public final class Application implements AutoCloseable { List<MlModelImporter> modelImporters = List.of(new VespaImporter(), new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); DeployState deployState = new DeployState.Builder() .applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), true)) .modelImporters(modelImporters) .deployLogger((level, s) -> { }) + .accessLoggingEnabledByDefault(false) .build(); return new VespaModel(new NullConfigModelRegistry(), deployState); } catch (IOException | SAXException e) { @@ -584,6 +587,7 @@ public final class Application implements AutoCloseable { xml.println("</search>"); } + xml.println("<accesslog type=\"disabled\" />"); xml.println("</container>"); } diff --git a/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/application/src/test/app-packages/searcher-app/services.xml b/application/src/test/app-packages/searcher-app/services.xml index 0fd37554c3b..5b9f4265b5a 100644 --- a/application/src/test/app-packages/searcher-app/services.xml +++ b/application/src/test/app-packages/searcher-app/services.xml @@ -12,4 +12,5 @@ </chain> </search> + <accesslog type="disabled" /> </container> diff --git a/application/src/test/app-packages/withcontent/services.xml b/application/src/test/app-packages/withcontent/services.xml index 251f591b6b8..4338579b49a 100644 --- a/application/src/test/app-packages/withcontent/services.xml +++ b/application/src/test/app-packages/withcontent/services.xml @@ -22,7 +22,7 @@ </chain> </document-processing> --> - + <accesslog type="disabled" /> </container> <content version="1.0" id="foo"> diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index f2af5537490..2c0b44c6d64 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -183,6 +183,7 @@ public class ApplicationTest { } @Test + // TODO: Creates access log public void renderer() throws Exception { try ( ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() @@ -375,7 +376,8 @@ public class ApplicationTest { private static String servicesXmlWithServer(int port) { return "<container version='1.0'>" + " <http> <server port='" + port +"' id='foo'/> </http>" + - "</container>"; + " <accesslog type=\"disabled\" />" + + "</container>"; } @Test @@ -392,7 +394,8 @@ public class ApplicationTest { " <access-control domain='foo' />" + " </filtering>" + " </http>" + - "</container>"; + " <accesslog type=\"disabled\" />" + + "</container>"; } } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java index 529f6a60c0d..81f5a681f1a 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java @@ -44,6 +44,7 @@ public class ContainerDocprocTest { xml += " </chain>\n" + " </document-processing>\n" + + " <accesslog type=\"disabled\" />" + "</container>\n"; return xml; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java index 79510375414..3d7eed1e729 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java @@ -45,7 +45,7 @@ public class ContainerModelEvaluationTest { } private void assertLoadedModels(JDisc jdisc) { { - String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\"}"; + String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; assertResponse("http://localhost/model-evaluation/v1", expected, jdisc); } @@ -55,6 +55,11 @@ public class ContainerModelEvaluationTest { } { + String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval", expected, jdisc); + } + + { // Note: The specific response value here has not been verified String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.5066885003407351},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3912837743150205},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":-0.12401806321703948},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.7019029168606575},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.13120114146441697},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":0.6611923203384626},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.22365810810026446},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":-0.0740018307465809},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":0.056492490256153896},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.18422015072393733}]}"; assertResponse("http://localhost/model-evaluation/v1/tensorflow_mnist_saved/serving_default.y/eval?input=" + inputTensor(), expected, jdisc); diff --git a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java index 93ca09ac5fc..22c307c0a0c 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java @@ -31,6 +31,7 @@ public class ContainerProcessingTest { xml += " </chain>\n" + " </processing>\n" + + " <accesslog type=\"disabled\" />" + "</container>\n"; return xml; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java index 8f3e7693bc5..a66547e6ed9 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java @@ -33,6 +33,7 @@ public class ContainerRequestTest { binding + "</binding>\n" + " </handler>\n" + + " <accesslog type=\"disabled\" />" + "</container>"; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java index d133b71b8da..793ee9ab094 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java @@ -8,6 +8,8 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import org.junit.Test; +import java.nio.charset.StandardCharsets; + import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -25,7 +27,7 @@ public class ContainerSearchTest { try (JDisc container = containerWithSearch(searcherId)) { byte[] rendered = container.search().processAndRender(ComponentSpecification.fromString("mychain"), ComponentSpecification.fromString("XmlRenderer"), new Query("")); - String renderedAsString = new String(rendered, "utf-8"); + String renderedAsString = new String(rendered, StandardCharsets.UTF_8); assertThat(renderedAsString, containsString(searcherId)); } } @@ -44,11 +46,12 @@ public class ContainerSearchTest { public JDisc containerWithSearch(String searcherId) { return JDisc.fromServicesXml("<container version=\"1.0\">" + // - "<search>" + // - "<chain id=\"mychain\">" + // - "<searcher id=\"" + searcherId + "\"/>" + // - "</chain>" + // - "</search>" + // + " <search>" + // + " <chain id=\"mychain\">" + // + " <searcher id=\"" + searcherId + "\"/>" + // + " </chain>" + // + " </search>" + // + " <accesslog type=\"disabled\" />" + // "</container>", Networking.disable); } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java index c284087243d..63347c2475a 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java @@ -164,6 +164,7 @@ public class ContainerTest { "<http>\n" + // "<server id=\"main\" port=\"9999\" />\n" + // "</http>\n" + // + "<accesslog type=\"disabled\" />" + "</container>"; return JDisc.fromServicesXml(xml, Networking.disable); } diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java index 0eb308f124c..caddabff8ed 100644 --- a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java +++ b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java @@ -66,11 +66,10 @@ public class JerseyTest { @Test public void jersey_resources_in_provided_dependencies_can_be_invoked_from_application() throws Exception { - BundleClasspathMapping providedDependency = new BundleClasspathMapping(bundleSymbolicName, - Arrays.asList(testClassesDirectory)); + BundleClasspathMapping providedDependency = + new BundleClasspathMapping(bundleSymbolicName, List.of(testClassesDirectory)); - save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), - Arrays.asList(providedDependency))); + save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), List.of(providedDependency))); with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); } @@ -113,16 +112,16 @@ public class JerseyTest { } private interface ThrowingConsumer<T> { - public void accept(T arg) throws Exception; + void accept(T arg) throws Exception; } private interface HttpGetter { - public HttpResponse get(String path) throws Exception; + HttpResponse get(String path) throws Exception; } @SuppressWarnings("try") // jdisc unreferenced inside try private void with_jersey_resources(List<String> packagesToScan, ThrowingConsumer<HttpGetter> f) throws Exception { - StringBuffer packageElements = new StringBuffer(); + StringBuilder packageElements = new StringBuilder(); for (String p : packagesToScan) { packageElements.append("<package>"); packageElements.append(p); @@ -140,6 +139,7 @@ public class JerseyTest { "<http>" + // "<server id=\"mainServer\" port=\"0\" />" + // "</http>" + // + "<accesslog type=\"disabled\" />" + "</container>" + // "</services>", // Networking.enable)) { @@ -177,8 +177,8 @@ public class JerseyTest { } public void saveMainBundleClassPathMappings(String classPathElement) throws Exception { - BundleClasspathMapping mainBundleClassPathMappings = new BundleClasspathMapping(bundleSymbolicName, - Arrays.asList(classPathElement)); + BundleClasspathMapping mainBundleClassPathMappings = + new BundleClasspathMapping(bundleSymbolicName, List.of(classPathElement)); save(new ProjectBundleClassPaths(mainBundleClassPathMappings, emptyList())); } diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 212c66a3431..f15ebe87b03 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -133,12 +133,18 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>flags</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index a1984557c31..466a9783bb3 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -17,7 +17,7 @@ import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index d164555dc3e..0ac833b7aee 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 3ea3892e398..4b882023e08 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -64,6 +64,7 @@ public class IdentityDocumentGeneratorTest { Optional.empty(), Optional.empty(), new MockNodeFlavors().getFlavorOrThrow("default"), + Optional.empty(), NodeType.host); Node containerNode = Node.createDockerNode(Set.of("::1"), containerHostname, diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index a35dfd878c5..2208b470ed8 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -223,7 +223,14 @@ public class InstanceValidatorTest { MockNodeFlavors flavors = new MockNodeFlavors(); List<Node> nodeList = new ArrayList<>(); for (int i = 0; i < num; i++) { - Node node = Node.create("foo" + i, new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), "foo" + i, Optional.empty(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node = Node.create("foo" + i, + new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), + "foo" + i, + Optional.empty(), + Optional.empty(), + flavors.getFlavorOrThrow("default"), + Optional.empty(), + NodeType.tenant); nodeList.add(node); } return nodeList; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java index c63ed3fceba..343a9feeed6 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java @@ -12,7 +12,7 @@ import com.yahoo.text.StringUtilities; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpUriRequest; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index 0eda6bd946b..6783763b210 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -54,6 +54,7 @@ public class ContainerTester { private static String servicesXml() { return "<container version='1.0'>\n" + + " <accesslog type=\"disabled\"/>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>10</maxthreads>\n" + " </config>\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index f04e155aa2c..99d1b064a34 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 5e3e2a42ed5..27c7ed6c21b 100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -55,7 +55,7 @@ cmake3 \ -DJAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-openjdk} \ -DCMAKE_PREFIX_PATH="/opt/vespa-deps" \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-deps/lib64;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION" \ + -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION;/usr/include/openblas" \ -DCMAKE_INSTALL_RPATH="${VESPA_INSTALL_PREFIX}/lib64;/opt/vespa-deps/lib64;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ ${UNPRIVILEGED_ARGS} \ ${EXTRA_CMAKE_ARGS} \ diff --git a/build_settings.cmake b/build_settings.cmake index b1b0b065771..53fcbbad9a2 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # @author Vegard Sjonfjell -include(vtag.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/vtag.cmake) # Build options # Whether to build unit tests as part of the 'all' target @@ -46,7 +46,7 @@ else() endif() # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} -mtune=intel ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}") else() diff --git a/client/README.md b/client/README.md index ddea3591e38..d08b19a1a12 100644 --- a/client/README.md +++ b/client/README.md @@ -1,12 +1,12 @@ # vespa_query_dsl This lib is used for composing vespa YQL queries -referece: https://docs.vespa.ai/documentation/reference/query-language-reference.html +Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html # usage -please refer the unit test: +Please refer to the unit test: -https://github.com/vespa-engine/vespa/blob/master/client/src/test/groovy/com/yahoo/vespa/client/dsl/QTest.groovy +https://github.com/vespa-engine/vespa/tree/master/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy # todos - [ ] support `predicate` (https://docs.vespa.ai/documentation/predicate-fields.html) diff --git a/client/pom.xml b/client/pom.xml index 3dee66f31ec..6f281fddf8c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -63,6 +63,21 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <finalName>${project.artifactId}-jar-with-dependencies</finalName> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/component/abi-spec.json b/component/abi-spec.json index a6b6a558db6..cd42b8ff95a 100644 --- a/component/abi-spec.json +++ b/component/abi-spec.json @@ -73,6 +73,7 @@ "public static com.yahoo.component.ComponentId fromString(java.lang.String)", "public java.lang.String toFileName()", "public static com.yahoo.component.ComponentId fromFileName(java.lang.String)", + "public static void resetGlobalCountersForTests()", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields": [] diff --git a/component/src/main/java/com/yahoo/component/ComponentId.java b/component/src/main/java/com/yahoo/component/ComponentId.java index 05c710e3fc1..4613be09543 100644 --- a/component/src/main/java/com/yahoo/component/ComponentId.java +++ b/component/src/main/java/com/yahoo/component/ComponentId.java @@ -32,15 +32,15 @@ public final class ComponentId implements Comparable<ComponentId> { private int count = 0; public int getAndIncrement() { return count++; } } - private static ThreadLocal<Counter> gid = new ThreadLocal<Counter>() { + private static ThreadLocal<Counter> threadLocalUniqueId = new ThreadLocal<Counter>() { @Override protected Counter initialValue() { return new Counter(); } }; - private static AtomicInteger uniqueTid = new AtomicInteger(0); - private static ThreadLocal<String> tid = new ThreadLocal<String>() { + private static AtomicInteger threadIdCounter = new AtomicInteger(0); + private static ThreadLocal<String> threadId = new ThreadLocal<String>() { @Override protected String initialValue() { - return new String("_"+uniqueTid.getAndIncrement()+"_"); + return new String("_" + threadIdCounter.getAndIncrement() + "_"); } }; @@ -58,7 +58,7 @@ public final class ComponentId implements Comparable<ComponentId> { } private String createAnonymousName(String name) { - return new StringBuilder(name).append(tid.get()).append(gid.get().getAndIncrement()).toString(); + return new StringBuilder(name).append(threadId.get()).append(threadLocalUniqueId.get().getAndIncrement()).toString(); } public ComponentId(String name, Version version, ComponentId namespace) { @@ -148,10 +148,7 @@ public final class ComponentId implements Comparable<ComponentId> { return spec.compareTo(other.spec); } - /** - * Creates a componentId that is unique for this run-time instance - */ - // TODO: Check if we really need this. -JB + /** Creates a componentId that is unique for this run-time instance */ public static ComponentId createAnonymousComponentId(String baseName) { return new ComponentId(baseName, null, null, true); } @@ -196,27 +193,27 @@ public final class ComponentId implements Comparable<ComponentId> { * Creates an id from a file <b>first</b> name string encoded in the standard translation (see {@link #toFileName}). * <b>Note</b> that any file last name, like e.g ".xml" must be stripped off before handoff to this method. */ - public static ComponentId fromFileName(final String fileName) { + public static ComponentId fromFileName(String fileName) { // Initial assumptions - String id=fileName; - Version version =null; - ComponentId namespace=null; + String id = fileName; + Version version = null; + ComponentId namespace = null; // Split out namespace, if any - int at=id.indexOf("@"); - if (at>0) { - String newId=id.substring(0,at); - namespace=ComponentId.fromString(id.substring(at+1)); - id=newId; + int at = id.indexOf("@"); + if (at > 0) { + String newId = id.substring(0, at); + namespace = ComponentId.fromString(id.substring(at + 1)); + id = newId; } // Split out version, if any - int dash=id.lastIndexOf("-"); - if (dash>0) { - String newId=id.substring(0,dash); + int dash = id.lastIndexOf("-"); + if (dash > 0) { + String newId = id.substring(0, dash); try { - version=new Version(id.substring(dash+1)); - id=newId; + version = new Version(id.substring(dash + 1)); + id = newId; } catch (IllegalArgumentException e) { // don't interpret the text following the dash as a version @@ -229,4 +226,10 @@ public final class ComponentId implements Comparable<ComponentId> { return new ComponentId(id,version,namespace); } + /** WARNING: For testing only: Resets counters creating anonymous component ids for this thread. */ + public static void resetGlobalCountersForTests() { + threadId.set("_0_"); + threadLocalUniqueId.set(new Counter()); + } + } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 3c1c6d87ea4..43527335802 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -758,6 +758,37 @@ ], "fields": [] }, + "com.yahoo.config.model.api.EndpointCertificateMetadata": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, int)", + "public java.lang.String keyName()", + "public java.lang.String certName()", + "public int version()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.EndpointCertificateSecrets": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public boolean isMissing()" + ], + "fields": [ + "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING" + ] + }, "com.yahoo.config.model.api.FileDistribution": { "superClass": "java.lang.Object", "interfaces": [], @@ -846,8 +877,10 @@ "public boolean useDedicatedNodeForLogserver()", "public abstract boolean useAdaptiveDispatch()", "public java.util.Optional tlsSecrets()", + "public java.util.Optional endpointCertificateSecrets()", "public abstract double defaultTermwiseLimit()", - "public abstract boolean useBucketSpaceMetric()" + "public abstract boolean useBucketSpaceMetric()", + "public boolean useNewAthenzFilter()" ], "fields": [] }, @@ -1003,6 +1036,7 @@ ], "methods": [ "public void <init>(java.lang.String, java.lang.String)", + "public void <init>(com.yahoo.config.model.api.EndpointCertificateSecrets)", "public java.lang.String certificate()", "public java.lang.String key()", "public boolean isMissing()" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java new file mode 100644 index 00000000000..a1fae9bb148 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java @@ -0,0 +1,35 @@ +package com.yahoo.config.model.api; + +public class EndpointCertificateMetadata { + + private final String keyName; + private final String certName; + private final int version; + + public EndpointCertificateMetadata(String keyName, String certName, int version) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + } + + public String keyName() { + return keyName; + } + + public String certName() { + return certName; + } + + public int version() { + return version; + } + + @Override + public String toString() { + return "EndpointCertificateMetadata{" + + "keyName='" + keyName + '\'' + + ", certName='" + certName + '\'' + + ", version=" + version + + '}'; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java new file mode 100644 index 00000000000..6fcbac4f422 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java @@ -0,0 +1,30 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +public class EndpointCertificateSecrets { + public static final EndpointCertificateSecrets MISSING = new EndpointCertificateSecrets(); + + private final String certificate; + private final String key; + + private EndpointCertificateSecrets() { + this(null, null); + } + + public EndpointCertificateSecrets(String certificate, String key) { + this.certificate = certificate; + this.key = key; + } + + public String certificate() { + return certificate; + } + + public String key() { + return key; + } + + public boolean isMissing() { + return this == MISSING; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 323aa473580..2410de55f86 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,10 +54,12 @@ public interface ModelContext { // TODO: Remove when Vespa 7.112 is the oldest config model in use default boolean useDedicatedNodeForLogserver() { return true; } boolean useAdaptiveDispatch(); - // TODO: Remove temporary default implementation + // TODO: Remove temporary default implementations default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } + default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); + default boolean useNewAthenzFilter() { return false; } } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 6a8b5a237ab..0937b8b77ec 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -16,6 +16,11 @@ public class TlsSecrets { this.key = key; } + public TlsSecrets(EndpointCertificateSecrets endpointCertificateSecrets) { + this.certificate = endpointCertificateSecrets.certificate(); + this.key = endpointCertificateSecrets.key(); + } + public String certificate() { return certificate; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index b286b94c699..a9b4e06ae1b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -15,7 +15,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -67,6 +67,7 @@ public class DeployState implements ConfigDefinitionStore { private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; private final Optional<Model> previousModel; + private final boolean accessLoggingEnabledByDefault; private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; @@ -101,14 +102,15 @@ public class DeployState implements ConfigDefinitionStore { Version vespaVersion, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, + Optional<Model> previousModel, Set<ContainerEndpoint> endpoints, Collection<MlModelImporter> modelImporters, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now, - Version wantedNodeVespaVersion) { + Version wantedNodeVespaVersion, + boolean accessLoggingEnabledByDefault) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -116,6 +118,7 @@ public class DeployState implements ConfigDefinitionStore { this.properties = properties; this.vespaVersion = vespaVersion; this.previousModel = previousModel; + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); this.documentModel = searchDocumentModel.getDocumentModel(); @@ -217,6 +220,10 @@ public class DeployState implements ConfigDefinitionStore { return logger; } + public boolean getAccessLoggingEnabledByDefault() { + return accessLoggingEnabledByDefault; + } + public FileRegistry getFileRegistry() { return fileRegistry; } @@ -255,7 +262,7 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public Optional<TlsSecrets> tlsSecrets() { return properties.tlsSecrets(); } + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); } public Optional<String> tlsClientAuthority() { var caFile = applicationPackage.getClientSecurityFile(); @@ -289,7 +296,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); + private boolean accessLoggingEnabledByDefault = true; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -361,6 +368,15 @@ public class DeployState implements ConfigDefinitionStore { return this; } + /** + * Whether access logging is enabled for an application without an accesslog element in services.xml. + * True by default. + */ + public Builder accessLoggingEnabledByDefault(boolean accessLoggingEnabledByDefault) { + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; + return this; + } + public DeployState build() { return build(new ValidationParameters()); } @@ -387,7 +403,8 @@ public class DeployState implements ConfigDefinitionStore { queryProfiles, semanticRules, now, - wantedNodeVespaVersion); + wantedNodeVespaVersion, + accessLoggingEnabledByDefault); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9d561a79c75..0b4562ecd5c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -4,6 +4,7 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; @@ -39,7 +40,8 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; private double defaultTermwiseLimit = 1.0; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); + private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); + private boolean useNewAthenzFilter = false; @Override public boolean multitenant() { return multitenant; } @@ -56,9 +58,11 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } - @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } + @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } + @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; @@ -95,9 +99,18 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + this.endpointCertificateSecrets = endpointCertificateSecrets; + return this; + } + + public TestProperties setUseNewAthenzFilter(boolean useNewAthenzFilter) { + this.useNewAthenzFilter = useNewAthenzFilter; + return this; + } - public TestProperties setTlsSecrets(Optional<TlsSecrets> tlsSecrets) { - this.tlsSecrets = tlsSecrets; + public TestProperties setZone(Zone zone) { + this.zone = zone; return this; } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 48c21f370f4..cce5a7850a0 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.log.LogLevel; -import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -21,14 +20,8 @@ import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.utils.FreezableMap; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -291,60 +284,6 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce public AbstractConfigProducer getParent() { return parent; } - /** - * Writes files that need to be written. The files will usually only be - * written when the Vespa model is generated through the deploy-application - * script. - * - * TODO: Make sure all implemented ConfigProducers call createConfig() - * instead of getConfig() when implementing this method. - */ - public void writeFiles(File directory) throws java.io.IOException { - if (!directory.isDirectory() && !directory.mkdirs()) { - throw new java.io.IOException("Cannot create directory: "+ directory); - } - for (Method m : getClass().getMethods()) { - try { - ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m); - if (builder!=null) { - writeBuilder(directory, m, builder); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - private void writeBuilder(File directory, Method m, - ConfigInstance.Builder builder) throws IllegalAccessException, - InvocationTargetException, InstantiationException, - NoSuchMethodException, IOException { - m.invoke(this, builder); - Class<?> configInstClass = builder.getClass().getEnclosingClass(); - ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder); - List<String> payloadList = ConfigInstance.serialize(inst); - File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg"); - FileOutputStream out = new FileOutputStream(outfn); - for (String s : payloadList) { - out.write(Utf8.toBytes(s)); - out.write('\n'); - } - } - - /** - * New Builder instance if m is getConfig(SomeConfig.Builder), or null - */ - private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws ReflectiveOperationException { - if (!"getConfig".equals(m.getName())) return null; - Type[] params = m.getParameterTypes(); - if (params.length!=1) return null; - Type param = params[0]; - if (!(param instanceof Class)) return null; - Class<?> paramClass = (Class<?>) param; - if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null; - return (ConfigInstance.Builder) paramClass.getDeclaredConstructor().newInstance(); - } - public void dump(PrintStream out) { for (ConfigProducer c : getChildren().values()) { out.println("id: " + c.getConfigId()); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set<String> importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set<Name> documentReferences) { + public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set<Name> documentReferences) { + Set<Name> documentReferences, + Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 9ff749a994c..ade8ae21870 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -41,7 +41,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { } if (structUsedByField.getName().equals(structType.getName())) { //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0); + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..ee4fc41f2f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,33 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; + +import java.util.Collections; +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List<Search> searchDefinitions; + + public ImportedFieldsEnumerator(List<Search> searchDefinitions) { + this.searchDefinitions = searchDefinitions; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.searchDefinitions.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index e46db1d1b5f..90f061d933d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.document.Stemming; @@ -9,8 +10,11 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; import java.util.Set; + /** * An index definition in a search definition. * Two indices are equal if they have the same name and the same settings, except @@ -57,6 +61,8 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; + private Optional<HnswIndexParams> hnswIndexParams; + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -115,20 +121,25 @@ public class Index implements Cloneable, Serializable { } @Override - public int hashCode() { - return name.hashCode() + ( prefix ? 17 : 0 ); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Index index = (Index) o; + return prefix == index.prefix && + normalized == index.normalized && + interleavedFeatures == index.interleavedFeatures && + Objects.equals(name, index.name) && + rankType == index.rankType && + Objects.equals(aliases, index.aliases) && + stemming == index.stemming && + type == index.type && + Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override - public boolean equals(Object object) { - if ( ! (object instanceof Index)) return false; - - Index other=(Index)object; - return - this.name.equals(other.name) && - this.prefix==other.prefix && - this.stemming==other.stemming && - this.normalized==other.normalized; + public int hashCode() { + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -176,6 +187,14 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<HnswIndexParams> getHnswIndexParams() { + return hnswIndexParams; + } + + public void setHnswIndexParams(HnswIndexParams params) { + hnswIndexParams = Optional.of(params); + } + public void setInterleavedFeatures(boolean value) { interleavedFeatures = value; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index 172e538d708..6de7c985326 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -70,6 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement public void setType(Reference reference, TensorType type) { featureTypes.put(reference, type); + queryFeaturesNotDeclared.remove(reference); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index f6aed1e5abc..23eb814de81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -776,7 +776,7 @@ public class RankProfile implements Cloneable { getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); rankingConstants().asMap().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); - // Add query features from rank profile types reached from the "default" profile + // Add query features from all rank profile types for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { for (FieldDescription field : queryProfileType.declaredFields().values()) { TensorType type = field.getType().asTensorType(); @@ -788,8 +788,7 @@ public class RankProfile implements Cloneable { type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + " with type " + field.getType().asTensorType() + - ", but this is already defined " + - "in another query profile with type " + + ", but this is already defined in another query profile with type " + context.getType(feature.get()))); context.setType(feature.get(), type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 76dff404568..8de8b239c93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,6 +240,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + aaB.index(ib); + } return aaB; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 245913d7822..fc8710fa1a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -15,9 +15,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; +import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; +import java.util.logging.Level; /** * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when @@ -39,12 +41,13 @@ public class DerivedConfiguration { private VsmSummary streamingSummary; private IndexSchema indexSchema; private ImportedFields importedFields; + private QueryProfileRegistry queryProfiles; /** * Creates a complete derived configuration from a search definition. * Only used in tests. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this argument is * live. Which means that this object will be inconsistent when the given search definition is later * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} @@ -56,11 +59,11 @@ public class DerivedConfiguration { /** * Creates a complete derived configuration snapshot from a search definition. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this * argument is live. Which means that this object will be inconsistent when the given * search definition is later modified. * @param deployLogger a {@link DeployLogger} for logging when doing operations on this - * @param deployProperties Properties set on deploy. + * @param deployProperties properties set on deploy * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles the query profiles of this application */ @@ -72,6 +75,7 @@ public class DerivedConfiguration { ImportedMlModels importedModels) { Validator.ensureNotNull("Search definition", search); this.search = search; + this.queryProfiles = queryProfiles; if ( ! search.isDocumentsOnly()) { streamingFields = new VsmFields(search); streamingSummary = new VsmSummary(search); @@ -120,6 +124,10 @@ public class DerivedConfiguration { exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); } + public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException { + exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); + } + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; try { @@ -186,4 +194,7 @@ public class DerivedConfiguration { public ImportedFields getImportedFields() { return importedFields; } + + public QueryProfileRegistry getQueryProfiles() { return queryProfiles; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 02b2a383318..032f7f58e2a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -286,16 +286,8 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { - // Explicit query commands on the field set, overrides everything. - if (!fieldSet.queryCommands().isEmpty()) { - for (String qc : fieldSet.queryCommands()) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(qc)); - } - return; - } + for (String qc : fieldSet.queryCommands()) + iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc)); boolean anyIndexing = false; boolean anyAttributing = false; boolean anyLowerCasing = false; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index 60b8ee78c7b..39a67a69575 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; @@ -20,7 +21,7 @@ import java.util.List; import java.util.Map; /** - * Deriver of indexschema config containing information of all index fields with name and data type. + * Deriver of indexschema config containing information of all text index fields with name and data type. * * @author geirst */ @@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { super.derive(search); } + private boolean isTensorField(ImmutableSDField field) { + return field.getDataType() instanceof TensorDataType; + } + private void deriveIndexFields(ImmutableSDField field, Search search) { - if (!field.doesIndexing() && - !field.isIndexStructureField()) + // Note: Indexes for tensor fields are NOT part of the index schema for text fields. + if ((!field.doesIndexing() && !field.isIndexStructureField()) || + isTensorField(field)) { return; } @@ -138,11 +144,10 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return Collections.singletonList(field); } if (fieldType instanceof ArrayDataType) { - boolean header = field.isHeader(); List<Field> ret = new LinkedList<>(); - Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header); + Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType()); for (Field flatField : flattenField(innerField)) { - ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header)); + ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()))); } return ret; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index fbcaf2a3a80..9ed5e4ca2de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -66,6 +66,8 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; + private Optional<HnswIndexParams> hnswIndexParams; + private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -150,6 +152,7 @@ public final class Attribute implements Cloneable, Serializable { setCollectionType(collectionType); this.tensorType = tensorType; this.referenceDocumentType = referenceDocumentType; + this.hnswIndexParams = Optional.empty(); } public Attribute convertToArray() { @@ -194,6 +197,7 @@ public final class Attribute implements Cloneable, Serializable { public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } @@ -217,6 +221,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } public Type getType() { return type; } @@ -335,7 +340,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, - isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType); + isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams); } @Override @@ -362,6 +367,7 @@ public final class Attribute implements Cloneable, Serializable { if ( ! this.sorting.equals(other.sorting)) return false; if (!this.tensorType.equals(other.tensorType)) return false; if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; + if (!this.hnswIndexParams.equals(other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java index 55dedc4a1d7..22424286ef9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java @@ -25,10 +25,7 @@ public class FieldSet { public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } public Set<String> getFieldNames() { return fieldNames; } public Set<ImmutableSDField> fields() { return fields; } - - public Set<String> queryCommands() { - return queryCommands; - } + public Set<String> queryCommands() { return queryCommands; } public void setMatching(Matching matching) { this.matching = matching; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java new file mode 100644 index 00000000000..70d0df8be7f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -0,0 +1,60 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.OptionalInt; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final OptionalInt maxLinksPerNode; + private final OptionalInt neighborsToExploreAtInsert; + + public static class Builder { + private OptionalInt maxLinksPerNode = OptionalInt.empty(); + private OptionalInt neighborsToExploreAtInsert = OptionalInt.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = OptionalInt.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = OptionalInt.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = OptionalInt.empty(); + this.neighborsToExploreAtInsert = OptionalInt.empty(); + } + + public HnswIndexParams(OptionalInt maxLinksPerNode, OptionalInt neighborsToExploreAtInsert) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(HnswIndexParams rhs) { + return new HnswIndexParams(rhs.maxLinksPerNode.isPresent() ? rhs.maxLinksPerNode : maxLinksPerNode, + rhs.neighborsToExploreAtInsert.isPresent() ? rhs.neighborsToExploreAtInsert : neighborsToExploreAtInsert); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 5b6fd7beb1e..30ce142d503 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -38,9 +38,8 @@ import java.util.TreeMap; /** * The field class represents a document field. It is used in - * the Document class to get and set fields. Each SDField has - * a name, a numeric ID, a data type, and a boolean that says whether it's - * a header field. The numeric ID is used when the fields are stored + * the Document class to get and set fields. Each SDField has a name, a numeric ID, + * a data type. The numeric ID is used when the fields are stored * in serialized form. * * @author bratseth @@ -120,15 +119,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * Creates a new field. This method is only used to create reserved fields * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) { - super(name, id, dataType, isHeader); - populate(populate, repo, name, dataType, isHeader); + protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + super(name, id, dataType); + populate(populate, repo, name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) { - this(repo, name, id, dataType, isHeader, true); + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { + this(repo, name, id, dataType, true); } /** @@ -136,41 +134,35 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, @param name The name of the field @param dataType The datatype of the field - @param isHeader Whether this is a "header" field or a "content" field - (true = "header"). */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) { - super(name,dataType,isHeader); - populate(populate, repo, name, dataType, isHeader); + public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { + super(name,dataType); + populate(populate, repo, name, dataType); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) { - populate(populate,repo, name, dataType, isHeader, null, 0); + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { + populate(populate,repo, name, dataType, null, 0); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) { + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, isHeader, recursion); + populateWithStructFields(repo, name, dataType, recursion); populateWithStructMatching(repo, name, dataType, fieldMatching); } } - public SDField(String name, DataType dataType, boolean isHeader) { - this(null, name, dataType, isHeader, true); - } /** * Creates a new field. * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner the owning document (used to check for id collisions) */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; - populate(populate, repo, name, dataType, isHeader); + populate(populate, repo, name, dataType); } /** @@ -178,27 +170,25 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner The owning document (used to check for id collisions) * @param fieldMatching The matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion); + populate(populate, repo, name, dataType, fieldMatching, recursion); } /** - * Constructor for <b>header</b> fields * * @param name The name of the field * @param dataType The datatype of the field */ public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name,dataType,true, true); + this(repo, name,dataType, true); } public SDField(String name, DataType dataType) { this(null, name,dataType); @@ -277,7 +267,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) { + public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { DataType dt = getFirstStructOrMapRecursive(); if (dt == null) { return; @@ -286,11 +276,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, MapDataType mdt = (MapDataType) dataType; SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("key", keyField); SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("value", valueField); } else { if (recursion >= 10) { @@ -306,7 +296,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } for (Field field : subType.fieldSet()) { SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - isHeader, subType, new Matching(), true, recursion + 1); + subType, new Matching(), true, recursion + 1); structFields.put(field.getName(), subField); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 886bf777d3a..04d11792379 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ import com.yahoo.document.DataType; */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) { - super(owner, name, dataType, isHeader, owner, false); + public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { + super(owner, name, dataType, owner, false); } public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, true, false); + super(null, name, dataType, false); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index 6fdf448a39b..a6707ec7ac0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -27,6 +27,7 @@ public class ExpressionTransforms { ImmutableList.of(new TensorFlowFeatureConverter(), new OnnxFeatureConverter(), new XgboostFeatureConverter(), + new LightGBMFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), new FunctionInliner(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java new file mode 100644 index 00000000000..5bde627dc0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java @@ -0,0 +1,59 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the lightgbm(model-path) pseudofeature with the + * native Vespa ranking expression implementing the same computation. + * + * @author lesters + */ +public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("lightgbm")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedLightGBMModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + + "the LightGBM model file under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java index 09e8b3ccdfa..ab9148992e0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java @@ -11,7 +11,7 @@ public interface FieldOperationContainer { /** Adds an operation */ void addOperation(FieldOperation op); - /** Apply all operations. Operations must be sorted in thjeir natural order before applying each operation. */ + /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */ void applyOperations(SDField field); String getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 39f543c7db3..7f9da28b9ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Index.Type; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -31,6 +32,8 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); + public String getIndexName() { return indexName; } @@ -91,6 +94,9 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (hnswIndexParams.isPresent()) { + index.setHnswIndexParams(hnswIndexParams.get().build()); + } } public Type getType() { @@ -116,8 +122,13 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } + public void setEnableBm25(boolean value) { enableBm25 = Optional.of(value); } + public void setHnswIndexParams(HnswIndexParams.Builder params) { + this.hnswIndexParams = Optional.of(params); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index d1eb18c4916..0ffd13927b4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -71,7 +71,7 @@ public class AddExtraFieldsToDocument extends Processor { if (docField == null) { ImmutableSDField existingField = search.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType(), true); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index 2d0998864f1..89b8889b4ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.logging.Level; /** @@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor { public void process(boolean validate, boolean documentsOnly) { if (documentsOnly) return; + Set<Reference> warnedAbout = new HashSet<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { - resolveTypesIn(profile, validate); + resolveTypesIn(profile, validate, warnedAbout); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e); @@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor { * * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types */ - private void resolveTypesIn(RankProfile profile, boolean validate) { + private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) { MapEvaluationTypeContext context = profile.typeContext(queryProfiles); for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { ExpressionFunction expressionFunction = function.getValue().function(); @@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor { profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) { - deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + - context.queryFeaturesNotDeclared()); + if ( context.tensorsAreUsed() && + ! context.queryFeaturesNotDeclared().isEmpty() && + ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { + deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() + + "' are not declared in query profile " + + "types and will be interpreted as scalars, not tensors: " + + context.queryFeaturesNotDeclared()); + warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 8e54d7c00d6..9cd7fb24e42 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -25,10 +26,11 @@ public class TensorFieldProcessor extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - for (SDField field : search.allConcreteFields()) { + for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { validateIndexingScripsForTensorField(field); validateAttributeSettingForTensorField(field); + processIndexSettingsForTensorField(field); } else if (field.getDataType() instanceof CollectionDataType){ validateDataTypeForCollectionField(field); @@ -37,20 +39,53 @@ public class TensorFieldProcessor extends Processor { } private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing()) { - fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); } } + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; + } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); + } + private void validateAttributeSettingForTensorField(SDField field) { if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); + var attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.isFastSearch()) { fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); } } } + private void processIndexSettingsForTensorField(SDField field) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (!field.doesAttributing()) { + fail(search, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null && index.getHnswIndexParams().isPresent()) { + params = params.overrideFrom(index.getHnswIndexParams().get()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + private void validateDataTypeForCollectionField(SDField field) { if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index d0a0bbfb748..d6398bc348c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ public class UriHack extends Processor { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType, true); + SDField partField = new SDField(partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java index bf86bc4a453..48cd378a9a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java @@ -1,16 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.List; -import java.util.Map; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.config.model.producer.UserConfigRepo; +import java.io.PrintStream; +import java.util.List; +import java.util.Map; + /** * Interface that should be implemented by all config producing modules * in the vespa model. @@ -35,17 +33,6 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer List<Service> getDescendantServices(); /** - * Writes files that need to be written. The files will usually - * only be written when the Vespa model is generated through the - * deploy-application script. - * This is primarily intended for debugging. - * - * @param directory directory to write files to - * @throws java.io.IOException if writing fails - */ - void writeFiles(File directory) throws IOException; - - /** * Dump the three of config producers to the specified stream. * * @param out The stream to print to, e.g. System.out diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a952d63526b..db469c2091e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import java.io.File; -import java.util.Objects; - import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; +import java.util.Objects; + /** * A physical host, running a set of services. * The identity of a host is its hostname. Hosts are comparable on their host name. @@ -67,10 +66,6 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> } @Override - public void writeFiles(File directory) { - } - - @Override public void getConfig(SentinelConfig.Builder builder) { // TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname> // as config id. We should probably wait for a major release diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 2cfd2e23d08..c8f708e84f4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -61,7 +61,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Constructor; @@ -564,23 +563,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri id2producer.put(configId, descendant); } - /** - * Writes MODEL.cfg files for all config producers. - * - * @param baseDirectory dir to write files to - */ - public void writeFiles(File baseDirectory) throws IOException { - super.writeFiles(baseDirectory); - for (ConfigProducer cp : id2producer.values()) { - try { - File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator)); - cp.writeFiles(destination); - } catch (IOException e) { - throw new IOException(cp.getConfigId() + ": " + e.getMessage()); - } - } - } - public Clients getClients() { return configModelRepo.getClients(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 08f782a0c54..fccacc3210d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; @@ -18,9 +21,8 @@ import java.util.LinkedHashMap; import java.util.Map; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_TYPE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler; /** * Container running a metrics proxy. @@ -29,14 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus */ public class MetricsProxyContainer extends Container implements NodeDimensionsConfig.Producer, + NodeInfoConfig.Producer, RpcConnectorConfig.Producer, VespaServicesConfig.Producer { - - static final class NodeDimensionNames { - static final String CLUSTER_TYPE = "clustertype"; - static final String CLUSTER_ID = "clusterid"; - } + public static final int BASEPORT = 19092; final boolean isHostedVespa; @@ -52,6 +51,7 @@ public class MetricsProxyContainer extends Container implements addMetricsProxyComponent(NodeDimensions.class); addMetricsProxyComponent(RpcConnector.class); addMetricsProxyComponent(VespaServices.class); + addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH)); } @Override @@ -59,8 +59,6 @@ public class MetricsProxyContainer extends Container implements return METRICS_PROXY_CONTAINER; } - static public int BASEPORT = 19092; - @Override public int getWantedPort() { return BASEPORT; @@ -119,15 +117,29 @@ public class MetricsProxyContainer extends Container implements Map<String, String> dimensions = new LinkedHashMap<>(); if (isHostedVespa) { getHostResource().spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { - dimensions.put(CLUSTER_TYPE, cluster.type().name()); - dimensions.put(CLUSTER_ID, cluster.id().value()); + dimensions.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); + dimensions.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); }); builder.dimensions(dimensions); } } - private void addMetricsProxyComponent(Class<?> componentClass) { + @Override + public void getConfig(NodeInfoConfig.Builder builder) { + builder.role(getNodeRole()) + .hostname(getHostName()); + } + + private String getNodeRole() { + String hostConfigId = getHost().getConfigId(); + if (! isHostedVespa) return hostConfigId; + return getHostResource().spec().membership() + .map(ClusterMembership::stringValue) + .orElse(hostConfigId); + } + + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index d7d178f13fc..071666b5bc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -16,6 +16,7 @@ import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; @@ -47,12 +48,10 @@ import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator. import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.INSTANCE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.SYSTEM; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.getDefaultPublicConsumer; import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer; @@ -79,8 +78,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC static final class AppDimensionNames { static final String SYSTEM = "system"; - static final String ZONE = "zone"; - static final String APPLICATION_ID = "applicationId"; // tenant.app.instance static final String TENANT = "tenantName"; static final String APPLICATION = "applicationName"; static final String INSTANCE = "instanceName"; @@ -122,11 +119,16 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + addComponent(metricsHandler); + } + + static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings("http://*" + bindingPath, "http://*" + bindingPath + "/*"); - addComponent(metricsHandler); + return metricsHandler; } @Override @@ -208,8 +210,8 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private Map<String, String> applicationDimensions() { Map<String, String> dimensions = new LinkedHashMap<>(); dimensions.put(SYSTEM, getZone().system().value()); - dimensions.put(ZONE, zoneString(getZone())); - dimensions.put(APPLICATION_ID, serializeWithDots(applicationId)); + dimensions.put(PublicDimensions.ZONE, zoneString(getZone())); + dimensions.put(PublicDimensions.APPLICATION_ID, serializeWithDots(applicationId)); dimensions.put(TENANT, applicationId.tenant().value()); dimensions.put(APPLICATION, applicationId.application().value()); dimensions.put(INSTANCE, applicationId.instance().value()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java index 19a62f47e39..6950eb1fd84 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java @@ -84,6 +84,7 @@ public class DefaultPublicMetrics { metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 57d938f8a71..e308be00faf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -72,6 +72,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.server.network.tls-connections-broken")); metrics.add(new Metric("vds.server.network.failed-tls-config-reloads")); + // C++ Fnet metrics + metrics.add(new Metric("vds.server.fnet.num-connections")); + return metrics; } @@ -453,10 +456,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); @@ -468,10 +474,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); @@ -518,8 +527,10 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); + metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count")); @@ -528,19 +539,27 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8 - + + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count")); @@ -560,13 +579,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); - //Distributor metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); @@ -596,18 +615,24 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); @@ -617,6 +642,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.max")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index b13fa4917e4..f029dad01a9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -62,7 +62,7 @@ public class MetricsBuilder { .collect(Collectors.toCollection(LinkedList::new)); List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream() - .map(metricSetElement -> availableMetricSets.get(metricSetElement.getAttribute(ID_ATTRIBUTE))) + .map(metricSetElement -> getMetricSetOrThrow(metricSetElement.getAttribute(ID_ATTRIBUTE))) .collect(Collectors.toCollection(LinkedList::new)); metricSets.add(defaultVespaMetricSet); @@ -75,6 +75,11 @@ public class MetricsBuilder { return "user-metrics-" + consumerName; } + private MetricSet getMetricSetOrThrow(String id) { + if (! availableMetricSets.containsKey(id)) throw new IllegalArgumentException("No such metric-set: " + id); + return availableMetricSets.get(id); + } + private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) { if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE) throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index 2f972b8ecb3..f00ad0f0dbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -1,17 +1,17 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.vespa.model.VespaModel; -public class TlsSecretsValidator extends Validator { +public class EndpointCertificateSecretsValidator extends Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override public void validate(VespaModel model, DeployState deployState) { - if (deployState.tlsSecrets().isPresent() && deployState.tlsSecrets().get() == TlsSecrets.MISSING) { + if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 8eabc61f71f..1e4a45428b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -57,7 +57,7 @@ public class Validation { new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index f14ad9c51a5..8fdcf249bbc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.model.VespaModel; import java.time.Instant; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -60,7 +61,7 @@ public class ResourcesReductionValidator implements ChangeValidator { private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) { // don't allow more than 50% reduction, but always allow to reduce by 1 if (nextValue >= currentValue * 0.5 || nextValue >= currentValue - 1) return Optional.empty(); - return Optional.of(String.format("Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); + return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); } private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 5e0dde6161d..58679c63565 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -1,15 +1,17 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; +import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.jdisc.http.ServletPathsConfig; @@ -18,8 +20,10 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; @@ -46,7 +50,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat RankProfilesConfig.Producer, RankingConstantsConfig.Producer, ServletPathsConfig.Producer, - ContainerMbusConfig.Producer { + ContainerMbusConfig.Producer, + MetricsProxyApiConfig.Producer { + + public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -55,7 +62,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private ContainerModelEvaluation modelEvaluation; - private Optional<TlsSecrets> tlsSecrets; private Optional<String> tlsClientAuthority; private MbusParams mbusParams; @@ -65,8 +71,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState); - - this.tlsSecrets = deployState.tlsSecrets(); this.tlsClientAuthority = deployState.tlsClientAuthority(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -76,6 +80,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); + addMetricsV2Handler(); addTestrunnerComponentsIfTester(deployState); } @@ -103,6 +108,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + public void addMetricsV2Handler() { + Handler<AbstractConfigProducer<?>> handler = new Handler<>( + new ComponentModel(METRICS_V2_HANDLER_CLASS, null, null, null)); + handler.addServerBindings("http://*" + MetricsV2Handler.V2_PATH, + "http://*" + MetricsV2Handler.V2_PATH + "/*"); + addComponent(handler); + } + private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); @@ -192,6 +205,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } @Override + public void getConfig(MetricsProxyApiConfig.Builder builder) { + builder.metricsPort(MetricsProxyContainer.BASEPORT) + .metricsApiPath(ApplicationMetricsHandler.VALUES_PATH); + } + + @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); builder.jvm.verbosegc(true) @@ -205,10 +224,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } - public Optional<TlsSecrets> getTlsSecrets() { - return tlsSecrets; - } - public Optional<String> getTlsClientAuthority() { return tlsClientAuthority; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java index c671749cff0..a466dabe984 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java @@ -4,7 +4,12 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.component.ComponentId; import com.yahoo.config.model.producer.AbstractConfigProducer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + /** * A group of config producers that have a component id. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index d3ba2718d71..6b1a94e16ae 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; @@ -34,7 +35,8 @@ public final class AccessControl { ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, ContainerCluster.STATE_HANDLER_CLASS, - ContainerCluster.LOG_HANDLER_CLASS + ContainerCluster.LOG_HANDLER_CLASS, + ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS ); public static final class Builder { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java index 4f84a01ff94..4a331718985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.List; import java.util.Optional; import static com.yahoo.component.ComponentSpecification.fromString; @@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString; * Configure SSL using file references * * @author mortent + * @author bjorncs */ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer { public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; @@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C private final String certificatePath; private final String caCertificatePath; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + private final List<String> cipherSuites; + private final List<String> protocolVersions; - public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + public ConfiguredFilebasedSslProvider(String servername, + String privateKeyPath, + String certificatePath, + String caCertificatePath, + String clientAuthentication, + List<String> cipherSuites, + List<String> protocolVersions) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C this.certificatePath = certificatePath; this.caCertificatePath = caCertificatePath; this.clientAuthentication = mapToConfigEnum(clientAuthentication); + this.cipherSuites = cipherSuites; + this.protocolVersions = protocolVersions; } @Override public void getConfig(ConnectorConfig.Builder builder) { - builder.ssl.enabled(true); - builder.ssl.privateKeyFile(privateKeyPath); - builder.ssl.certificateFile(certificatePath); - builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); - builder.ssl.clientAuth(clientAuthentication); + builder.ssl( + new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyPath) + .certificateFile(certificatePath) + .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")) + .clientAuth(clientAuthentication) + .enabledCipherSuites(cipherSuites) + .enabledProtocols(protocolVersions)); } public SimpleComponent getComponent() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 7a08a3c1a7b..0ad9bd9e883 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -17,21 +17,26 @@ import java.util.List; public class HostedSslConnectorFactory extends ConnectorFactory { private static final List<String> INSECURE_WHITELISTED_PATHS = List.of("/status.html"); + private static final String DEFAULT_HOSTED_TRUSTSTORE = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; private final boolean enforceClientAuth; /** - * Create connector factory that uses a certificate provided by the config-model / configserver. + * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ - public static HostedSslConnectorFactory withProvidedCertificate(String serverName, TlsSecrets tlsSecrets) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, /*tlsCaCertificates*/null), false); + // TODO Enforce client authentication + public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { + return new HostedSslConnectorFactory( + createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ - public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, tlsCaCertificates), true); + public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory( + createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true); } /** @@ -47,12 +52,12 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates) { return new ConfiguredDirectSslProvider( serverName, - tlsSecrets.key(), - tlsSecrets.certificate(), - /*caCertificatePath*/null, + endpointCertificateSecrets.key(), + endpointCertificateSecrets.certificate(), + tlsCaCertificatesPath, tlsCaCertificates, ClientAuth.Enum.WANT_AUTH); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index b37caf22216..b492941fd13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -139,13 +139,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> int legalPortInHostedVespa = Container.BASEPORT; if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) { - // TODO: After January 2020: - // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner - // - Enable 2 currently ignored tests in this module - // - throw IllegalArgumentException here instead of warning - logger.log(Level.WARNING, "Illegal port " + port + " in http server '" + - spec.stringAttribute("id") + "'" + - ": Port must be set to " + legalPortInHostedVespa); + throw new IllegalArgumentException("Illegal port " + port + " in http server '" + + spec.stringAttribute("id") + "'" + + ": Port must be set to " + legalPortInHostedVespa); } return port; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index db831a1ec2f..562026ab4dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider; +import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import org.w3c.dom.Element; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import static java.util.stream.Collectors.toList; + /** * @author Einar M R Rosenvinge * @author mortent @@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); + List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites"); + List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols"); return new ConfiguredFilebasedSslProvider( serverName, privateKeyFile, certificateFile, caCertificateFile.orElse(null), - clientAuthentication.orElse(null)); + clientAuthentication.orElse(null), + cipherSuites, + protocols); } else if (sslProviderConfigurator != null) { String className = sslProviderConfigurator.getAttribute("class"); String bundle = sslProviderConfigurator.getAttribute("bundle"); @@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil } } + private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) { + return XmlHelper.getOptionalChildValue(sslElement, listElementName) + .map(element -> + Arrays.stream(element.split(",")) + .filter(listEntry -> !listEntry.isBlank()) + .map(String::trim) + .collect(toList())) + .orElse(List.of()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index e19d81e7fb2..fa5fa4bd227 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,6 +8,7 @@ import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; @@ -58,7 +59,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) { for (AbstractSearchCluster searchCluster : searchClusters) { if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster)); + var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); + owningCluster.addComponent(dispatcher); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 704188e80e8..284aa3b46c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; @@ -13,7 +14,7 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster; * * @author bratseth */ -public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel> +public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel> implements DispatchConfig.Producer { private final IndexedSearchCluster indexedSearchCluster; @@ -21,14 +22,17 @@ public class DispatcherComponent extends Component<DispatcherComponent, Componen public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) { super(toComponentModel(indexedSearchCluster)); this.indexedSearchCluster = indexedSearchCluster; + String clusterName = indexedSearchCluster.getClusterName(); + var rpcResoucePool = new RpcResourcePoolComponent(clusterName); + inject(rpcResoucePool); + addComponent(rpcResoucePool); } private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) { String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, - "com.yahoo.search.dispatch.Dispatcher", - BundleMapper.searchAndDocprocBundle, - null); + com.yahoo.search.dispatch.Dispatcher.class.getName(), + BundleMapper.searchAndDocprocBundle); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 0abb0803405..0a9618e7b08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -232,8 +232,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer return propB; } - private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig( - String fullName, Object value) { + private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(String fullName, Object value) { QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); if (value instanceof SubstituteString) value=value.toString(); // Send only types understood by configBuilder downwards @@ -251,7 +250,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer qtB.matchaspath(true); for (QueryProfileType inherited : profileType.inherited()) qtB.inherit(inherited.getId().stringValue()); - List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values()); + List<FieldDescription> fields = new ArrayList<>(profileType.declaredFields().values()); Collections.sort(fields); for (FieldDescription field : fields) qtB.field(createConfig(field)); @@ -260,22 +259,20 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) { QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder(); - fB. - name(field.getName()). - type(field.getType().stringValue()); + fB.name(field.getName()).type(field.getType().stringValue()); if ( ! field.isOverridable()) fB.overridable(false); if (field.isMandatory()) fB.mandatory(true); - String aliases=toSpaceSeparatedString(field.getAliases()); - if (!aliases.isEmpty()) + String aliases = toSpaceSeparatedString(field.getAliases()); + if ( ! aliases.isEmpty()) fB.alias(aliases); return fB; } public String toSpaceSeparatedString(List<String> list) { - StringBuilder b=new StringBuilder(); - for (Iterator<String> i=list.iterator(); i.hasNext(); ) { + StringBuilder b = new StringBuilder(); + for (Iterator<String> i = list.iterator(); i.hasNext(); ) { b.append(i.next()); if (i.hasNext()) b.append(" "); @@ -290,10 +287,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer } } - /** - * The config produced by this - * @return query profiles config - */ + /** Returns the config produced by this */ public QueryProfilesConfig getConfig() { QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder(); getConfig(qB); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java new file mode 100644 index 00000000000..2689c2ce71b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -0,0 +1,18 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.BundleMapper; + +public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { + + public RpcResourcePoolComponent(String clusterName) { + super(toComponentModel(clusterName)); + } + + private static ComponentModel toComponentModel(String clusterName) { + String componentId = "rpcresourcepool." + clusterName; + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3da0b01f614..9d7274f1bbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -13,7 +13,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -302,7 +302,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } - if (accessLogElements.isEmpty() && cluster.getSearch() != null) + if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) cluster.addDefaultSearchAccessLog(); } @@ -327,15 +327,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - if (deployState.tlsSecrets().isPresent()) { + if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); } - TlsSecrets tlsSecrets = deployState.tlsSecrets().get(); + EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, tlsSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, tlsSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); server.addConnector(connectorFactory); } else { server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 60b3cb6987c..321564bfec1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -35,14 +35,6 @@ public abstract class SearchCluster extends AbstractSearchCluster super(parent, clusterName, index); } - public void writeFiles(File directory) throws java.io.IOException { - if (!directory.isDirectory() && !directory.mkdirs()) { - throw new java.io.IOException("Cannot create directory: "+ directory); - } - writeSdFiles(directory); - super.writeFiles(directory); - } - /** * Must be called after cluster is built, to derive SD configs * Derives the search definitions from the application package.. diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 52665ff56a9..0cfd13f2a9c 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -332,6 +332,9 @@ TOKEN : | < UPPERBOUND: "upper-bound" > | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > | < ENABLE_BM25: "enable-bm25" > +| < HNSW: "hnsw" > +| < MAXLINKSPERNODE: "max-links-per-node" > +| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > @@ -637,7 +640,7 @@ void field(SDDocumentType document, Search search) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, document); + field = new TemporarySDField(name, type, document); } lbrace() (fieldBody(field, search, document) (<NL>)*)* <RBRACE> { @@ -651,27 +654,35 @@ void field(SDDocumentType document, Search search) : void fieldSet(Search search) : { - String name; -} -{ - <FIELDSET> name = identifier() lbrace() - (fieldSetItem(name, search)(<NL>)*)+ - <RBRACE> -} - -void fieldSetItem(String setName, Search search) : -{ + String setName; String field; String queryCommand; - SDField matchSettings = new SDField(setName, DataType.STRING); // match etc for fieldset represented as SDField or ease of parsing + List queryCommands = new ArrayList(); + FieldOperationContainer matchSetting; + List matchSettings = new ArrayList(); } { - ( <FIELDS><COLON> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } - ( <COMMA> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* ) - | - ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { search.fieldSets().userFieldSets().get(setName).queryCommands().add(queryCommand);} - | - ( match(matchSettings) ) { matchSettings.applyOperations(); search.fieldSets().userFieldSets().get(setName).setMatching(matchSettings.getMatching());} + <FIELDSET> setName = identifier() lbrace() + (( + ( <FIELDS><COLON> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } + ( <COMMA> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* ) + | + ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } + | + ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } + )(<NL>)*)+ + <RBRACE> + { + // Apply settings after parsing since all user field items must be set first + + for (Object command : queryCommands) + search.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command); + + for (Object setting : matchSettings) { + ((SDField)setting).applyOperations(); + search.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching()); + } + } } /** @@ -923,7 +934,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, struct); + field = new TemporarySDField(name, type, struct); struct.addField(field); } lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { @@ -1531,7 +1542,7 @@ void queryCommand(FieldOperationContainer container) : QueryCommandOperation field = new QueryCommandOperation(); } { - <QUERYCOMMAND> <COLON> command = identifierWithDash() + <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() ) { field.addQueryCommand(command); container.addOperation(field); @@ -1551,11 +1562,11 @@ void alias(FieldOperationContainer container) : } } -Object match(FieldOperationContainer field) : { } +FieldOperationContainer match(FieldOperationContainer field) : { } { <MATCH> ( (<COLON> matchType(field)) | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) ) - { return null; } + { return field; } } /** @@ -1803,10 +1814,33 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | hnswIndex(index) { } ) { return null; } } +void hnswIndex(IndexOperation index) : +{ + HnswIndexParams.Builder params = new HnswIndexParams.Builder(); +} +{ + ( LOOKAHEAD(<HNSW> lbrace()) + <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) | + <HNSW> ) + { + index.setHnswIndexParams(params); + } +} + +void hnswIndexBody(HnswIndexParams.Builder params) : +{ + int num; +} +{ + ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } + | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) +} + /** * Consumes a constant block of a search element. * diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index e3ad942e7b3..c47983adc12 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -37,7 +37,8 @@ OptionalDedicatedNodes = element nodes { attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & attribute dedicated { xsd:boolean }? & - attribute exclusive { xsd:boolean }? + attribute exclusive { xsd:boolean }? & + Resources? } GenericConfig = element config { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 6e4346d96ee..726fa849c00 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -96,7 +96,9 @@ Ssl = element ssl { element private-key-file { string } & element certificate-file { string } & element ca-certificates-file { string }? & - element client-authentication { string "disabled" | string "want" | string "need" }? + element client-authentication { string "disabled" | string "want" | string "need" }? & + element cipher-suites { string }? & + element protocols { string }? } SslProvider = element ssl-provider { diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 8b3868c132e..bd09902f929 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -57,6 +57,9 @@ PersistenceThread = element thread { ## Declare which storage threads each disk should have. PersistenceThreads = element persistence-threads { + ## The number of threads to create + attribute count { xsd:integer }? & + ## All of the below settings are deprecated. ## Operations with priority worse than this can be blocked attribute highest-priority-to-block { xsd:string } ? & ## Operations with priority better than this can block others diff --git a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg b/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg deleted file mode 100644 index 6e18bdf64e7..00000000000 --- a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg +++ /dev/null @@ -1,3 +0,0 @@ -namespace=config -foo bar -baz []678 diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd index ab5e42f983d..247df8a0241 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd @@ -33,8 +33,12 @@ search test { expression: xgboost("xgboost_2_2") } + function my_lightgbm() { + expression: lightgbm("lightgbm_regression") + } + first-phase { - expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm } } diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index c5b1dfef610..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19125 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 7aed9bdc244..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19131 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg deleted file mode 100644 index 49e5f59b9be..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19137 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg deleted file mode 100644 index 8d5d4fdde7f..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19143 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index be7d8e44c16..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19110 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg deleted file mode 100644 index f39dc6adc03..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19112 -slobrok.name "search/cluster.music/rtx/1/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 5c9f46bf8ce..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19156 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 69f91eab48c..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19105 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg deleted file mode 100644 index eebb1cb6b40..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19111 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg deleted file mode 100644 index 82e9aafc5a8..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19162 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index aa557a9ae04..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19144 -slobrok.name "search/cluster.rt/rtx/0/clustercontroller" -slobrok.config search/cluster.rt/rtx -servicemonitor.autodisable true -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg deleted file mode 100644 index 1407c3ff209..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19146 -slobrok.name "search/cluster.rt/rtx/1/clustercontroller" -slobrok.config search/cluster.rt/rtx -servicemonitor.autodisable true -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index c5b1dfef610..00000000000 --- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19125 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index d0888e9be96..00000000000 --- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19115 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 53dcc3f9686..00000000000 --- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19118 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index 474b8c68cdb..00000000000 --- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19108 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 6dabdfc6af7..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19115 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index 3c1b537236a..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19105 -slobrok.name "search/cluster.music1/rtx/0/clustercontroller" -slobrok.config search/cluster.music1/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index b6d62fc678b..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19126 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 887eaa6b634..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19132 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index c00fcc456d6..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19116 -slobrok.name "search/cluster.music2/rtx/0/clustercontroller" -slobrok.config search/cluster.music2/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7b50176625d --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -0,0 +1,104 @@ +enablecompression false +datatype[0].id 1381038251 +datatype[0].structtype[0].name "position" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].compresstype NONE +datatype[0].structtype[0].compresslevel 0 +datatype[0].structtype[0].compressthreshold 95 +datatype[0].structtype[0].compressminsize 800 +datatype[0].structtype[0].field[0].name "x" +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" +datatype[0].structtype[0].field[1].name "y" +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" +datatype[1].id 595216861 +datatype[1].referencetype[0].target_type_id -1318255918 +datatype[2].id 542332920 +datatype[2].referencetype[0].target_type_id 443162583 +datatype[3].id 959075962 +datatype[3].structtype[0].name "ad.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "campaign_ref" +datatype[3].structtype[0].field[0].datatype 595216861 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "person_ref" +datatype[3].structtype[0].field[1].datatype 542332920 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[4].id -255288561 +datatype[4].structtype[0].name "ad.body" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id 2987301 +datatype[5].documenttype[0].name "ad" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct 959075962 +datatype[5].documenttype[0].bodystruct -255288561 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].documenttype[0].importedfield[0].name "my_cool_field" +datatype[5].documenttype[0].importedfield[1].name "my_swag_field" +datatype[5].documenttype[0].importedfield[2].name "my_name" +datatype[6].id -2041471955 +datatype[6].structtype[0].name "campaign.header" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].compresstype NONE +datatype[6].structtype[0].compresslevel 0 +datatype[6].structtype[0].compressthreshold 95 +datatype[6].structtype[0].compressminsize 800 +datatype[6].structtype[0].field[0].name "cool_field" +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].detailedtype "" +datatype[6].structtype[0].field[1].name "swag_field" +datatype[6].structtype[0].field[1].datatype 4 +datatype[6].structtype[0].field[1].detailedtype "" +datatype[7].id 1448849794 +datatype[7].structtype[0].name "campaign.body" +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].compresstype NONE +datatype[7].structtype[0].compresslevel 0 +datatype[7].structtype[0].compressthreshold 95 +datatype[7].structtype[0].compressminsize 800 +datatype[8].id -1318255918 +datatype[8].documenttype[0].name "campaign" +datatype[8].documenttype[0].version 0 +datatype[8].documenttype[0].inherits[0].name "document" +datatype[8].documenttype[0].inherits[0].version 0 +datatype[8].documenttype[0].headerstruct -2041471955 +datatype[8].documenttype[0].bodystruct 1448849794 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[9].id 3129224 +datatype[9].structtype[0].name "person.header" +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].compresstype NONE +datatype[9].structtype[0].compresslevel 0 +datatype[9].structtype[0].compressthreshold 95 +datatype[9].structtype[0].compressminsize 800 +datatype[9].structtype[0].field[0].name "name" +datatype[9].structtype[0].field[0].datatype 2 +datatype[9].structtype[0].field[0].detailedtype "" +datatype[10].id -2003767395 +datatype[10].structtype[0].name "person.body" +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].compresstype NONE +datatype[10].structtype[0].compresslevel 0 +datatype[10].structtype[0].compressthreshold 95 +datatype[10].structtype[0].compressminsize 800 +datatype[11].id 443162583 +datatype[11].documenttype[0].name "person" +datatype[11].documenttype[0].version 0 +datatype[11].documenttype[0].inherits[0].name "document" +datatype[11].documenttype[0].inherits[0].version 0 +datatype[11].documenttype[0].headerstruct 3129224 +datatype[11].documenttype[0].bodystruct -2003767395 +datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7859703ffe0 --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -0,0 +1,141 @@ +enablecompression false +documenttype[0].id 2987301 +documenttype[0].name "ad" +documenttype[0].version 0 +documenttype[0].headerstruct 959075962 +documenttype[0].bodystruct -255288561 +documenttype[0].inherits[0].id 8 +documenttype[0].datatype[0].id 959075962 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ad.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 95 +documenttype[0].datatype[0].sstruct.compression.minsize 200 +documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" +documenttype[0].datatype[0].sstruct.field[0].id 23963250 +documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" +documenttype[0].datatype[0].sstruct.field[1].name "person_ref" +documenttype[0].datatype[0].sstruct.field[1].id 100779805 +documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" +documenttype[0].datatype[1].id -255288561 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "ad.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 95 +documenttype[0].datatype[1].sstruct.compression.minsize 200 +documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +documenttype[0].referencetype[0].id 595216861 +documenttype[0].referencetype[0].target_type_id -1318255918 +documenttype[0].referencetype[1].id 542332920 +documenttype[0].referencetype[1].target_type_id 443162583 +documenttype[0].importedfield[0].name "my_cool_field" +documenttype[0].importedfield[1].name "my_swag_field" +documenttype[0].importedfield[2].name "my_name" +documenttype[1].id -1318255918 +documenttype[1].name "campaign" +documenttype[1].version 0 +documenttype[1].headerstruct -2041471955 +documenttype[1].bodystruct 1448849794 +documenttype[1].inherits[0].id 8 +documenttype[1].datatype[0].id -2041471955 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "campaign.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 95 +documenttype[1].datatype[0].sstruct.compression.minsize 200 +documenttype[1].datatype[0].sstruct.field[0].name "cool_field" +documenttype[1].datatype[0].sstruct.field[0].id 1588702436 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" +documenttype[1].datatype[0].sstruct.field[1].name "swag_field" +documenttype[1].datatype[0].sstruct.field[1].id 1691224741 +documenttype[1].datatype[0].sstruct.field[1].datatype 4 +documenttype[1].datatype[0].sstruct.field[1].detailedtype "" +documenttype[1].datatype[1].id 1448849794 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "campaign.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 95 +documenttype[1].datatype[1].sstruct.compression.minsize 200 +documenttype[1].fieldsets{[document]}.fields[0] "cool_field" +documenttype[1].fieldsets{[document]}.fields[1] "swag_field" +documenttype[2].id 443162583 +documenttype[2].name "person" +documenttype[2].version 0 +documenttype[2].headerstruct 3129224 +documenttype[2].bodystruct -2003767395 +documenttype[2].inherits[0].id 8 +documenttype[2].datatype[0].id 3129224 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "person.header" +documenttype[2].datatype[0].sstruct.version 0 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 95 +documenttype[2].datatype[0].sstruct.compression.minsize 200 +documenttype[2].datatype[0].sstruct.field[0].name "name" +documenttype[2].datatype[0].sstruct.field[0].id 1160796772 +documenttype[2].datatype[0].sstruct.field[0].datatype 2 +documenttype[2].datatype[0].sstruct.field[0].detailedtype "" +documenttype[2].datatype[1].id -2003767395 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "person.body" +documenttype[2].datatype[1].sstruct.version 0 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 95 +documenttype[2].datatype[1].sstruct.compression.minsize 200 +documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 97f480745cf..cf8644ebe83 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 56fd15f9f5d..29d5dd92043 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index 022bdbd31a4..773f796ed59 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 7a21001a9ed..e3faf7662f4 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index dda746da5b4..b4971487bd1 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,3 +211,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/fieldset2/index-info.cfg b/config-model/src/test/derived/fieldset2/index-info.cfg new file mode 100644 index 00000000000..7c3c1c448db --- /dev/null +++ b/config-model/src/test/derived/fieldset2/index-info.cfg @@ -0,0 +1,37 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "phrase-segmenting false" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "default" +indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/fieldset2/test.sd b/config-model/src/test/derived/fieldset2/test.sd new file mode 100644 index 00000000000..7606a5d16b2 --- /dev/null +++ b/config-model/src/test/derived/fieldset2/test.sd @@ -0,0 +1,20 @@ +search test { + + document test { + + field field1 type string { + indexing: index + } + + field field2 type string { + indexing: index + } + + } + + fieldset default { + fields: field1, field2 + query-command: "phrase-segmenting false" + } + +} diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg new file mode 100644 index 00000000000..27c9f1e0d13 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -0,0 +1,24 @@ +attribute[].name "t1" +attribute[].datatype TENSOR +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "tensor(x[128])" +attribute[].imported false +attribute[].index.hnsw.enabled true +attribute[].index.hnsw.maxlinkspernode 32 +attribute[].index.hnsw.neighborstoexploreatinsert 300 diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg new file mode 100644 index 00000000000..e9fc265ca67 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -0,0 +1,5 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "test" +ilscript[].docfield[] "t1" +ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }" diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd new file mode 100644 index 00000000000..03ede04208b --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -0,0 +1,13 @@ +search test { + document test { + field t1 type tensor(x[128]) { + indexing: attribute | index + index { + hnsw { + max-links-per-node: 32 + neighbors-to-explore-at-insert: 300 + } + } + } + } +} diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index db2280a7846..5427e856df0 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -1,42 +1,48 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_pos_zcurve" -attribute[1].datatype INT64 -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index ce6ff5e54ae..e1969e991dd 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -1,168 +1,192 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_elem_array.name" -attribute[1].datatype STRING -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true -attribute[2].name "my_elem_array.weight" -attribute[2].datatype INT32 -attribute[2].collectiontype SINGLE -attribute[2].removeifzero false -attribute[2].createifnonexistent false -attribute[2].fastsearch false -attribute[2].huge false -attribute[2].ismutable false -attribute[2].sortascending true -attribute[2].sortfunction UCA -attribute[2].sortstrength PRIMARY -attribute[2].sortlocale "" -attribute[2].enablebitvectors false -attribute[2].enableonlybitvector false -attribute[2].fastaccess false -attribute[2].arity 8 -attribute[2].lowerbound -9223372036854775808 -attribute[2].upperbound 9223372036854775807 -attribute[2].densepostinglistthreshold 0.4 -attribute[2].tensortype "" -attribute[2].imported true -attribute[3].name "my_elem_map.key" -attribute[3].datatype STRING -attribute[3].collectiontype SINGLE -attribute[3].removeifzero false -attribute[3].createifnonexistent false -attribute[3].fastsearch true -attribute[3].huge false -attribute[3].ismutable false -attribute[3].sortascending true -attribute[3].sortfunction UCA -attribute[3].sortstrength PRIMARY -attribute[3].sortlocale "" -attribute[3].enablebitvectors false -attribute[3].enableonlybitvector false -attribute[3].fastaccess false -attribute[3].arity 8 -attribute[3].lowerbound -9223372036854775808 -attribute[3].upperbound 9223372036854775807 -attribute[3].densepostinglistthreshold 0.4 -attribute[3].tensortype "" -attribute[3].imported true -attribute[4].name "my_elem_map.value.name" -attribute[4].datatype STRING -attribute[4].collectiontype SINGLE -attribute[4].removeifzero false -attribute[4].createifnonexistent false -attribute[4].fastsearch true -attribute[4].huge false -attribute[4].ismutable false -attribute[4].sortascending true -attribute[4].sortfunction UCA -attribute[4].sortstrength PRIMARY -attribute[4].sortlocale "" -attribute[4].enablebitvectors false -attribute[4].enableonlybitvector false -attribute[4].fastaccess false -attribute[4].arity 8 -attribute[4].lowerbound -9223372036854775808 -attribute[4].upperbound 9223372036854775807 -attribute[4].densepostinglistthreshold 0.4 -attribute[4].tensortype "" -attribute[4].imported true -attribute[5].name "my_elem_map.value.weight" -attribute[5].datatype INT32 -attribute[5].collectiontype SINGLE -attribute[5].removeifzero false -attribute[5].createifnonexistent false -attribute[5].fastsearch false -attribute[5].huge false -attribute[5].ismutable false -attribute[5].sortascending true -attribute[5].sortfunction UCA -attribute[5].sortstrength PRIMARY -attribute[5].sortlocale "" -attribute[5].enablebitvectors false -attribute[5].enableonlybitvector false -attribute[5].fastaccess false -attribute[5].arity 8 -attribute[5].lowerbound -9223372036854775808 -attribute[5].upperbound 9223372036854775807 -attribute[5].densepostinglistthreshold 0.4 -attribute[5].tensortype "" -attribute[5].imported true -attribute[6].name "my_str_int_map.key" -attribute[6].datatype STRING -attribute[6].collectiontype SINGLE -attribute[6].removeifzero false -attribute[6].createifnonexistent false -attribute[6].fastsearch true -attribute[6].huge false -attribute[6].ismutable false -attribute[6].sortascending true -attribute[6].sortfunction UCA -attribute[6].sortstrength PRIMARY -attribute[6].sortlocale "" -attribute[6].enablebitvectors false -attribute[6].enableonlybitvector false -attribute[6].fastaccess false -attribute[6].arity 8 -attribute[6].lowerbound -9223372036854775808 -attribute[6].upperbound 9223372036854775807 -attribute[6].densepostinglistthreshold 0.4 -attribute[6].tensortype "" -attribute[6].imported true -attribute[7].name "my_str_int_map.value" -attribute[7].datatype INT32 -attribute[7].collectiontype SINGLE -attribute[7].removeifzero false -attribute[7].createifnonexistent false -attribute[7].fastsearch false -attribute[7].huge false -attribute[7].ismutable false -attribute[7].sortascending true -attribute[7].sortfunction UCA -attribute[7].sortstrength PRIMARY -attribute[7].sortlocale "" -attribute[7].enablebitvectors false -attribute[7].enableonlybitvector false -attribute[7].fastaccess false -attribute[7].arity 8 -attribute[7].lowerbound -9223372036854775808 -attribute[7].upperbound 9223372036854775807 -attribute[7].densepostinglistthreshold 0.4 -attribute[7].tensortype "" -attribute[7].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.value" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 15b1771a2e8..168c9bd4659 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,3 +187,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index a83ec45c5e9..65818e088f5 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -44,6 +44,8 @@ indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sd" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "sd" +indexinfo[].command[].command "phrase-segmenting false" +indexinfo[].command[].indexname "sd" indexinfo[].command[].command "literal-boost" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "index" @@ -307,6 +309,16 @@ indexinfo[].command[].indexname "exactexplicit" indexinfo[].command[].command "exact ARNOLD" indexinfo[].command[].indexname "exactexplicit" indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "exactexplicit2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "exactexplicit2" diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd index 49f0f7dfca6..01f552a8c32 100644 --- a/config-model/src/test/derived/indexschema/indexschema.sd +++ b/config-model/src/test/derived/indexschema/indexschema.sd @@ -24,6 +24,7 @@ search indexschema { field sd type string { indexing: index rank:literal + query-command: "phrase-segmenting false" } field pos type position { indexing: attribute @@ -130,17 +131,17 @@ search indexschema { } fieldset exactexplicit { - fields:sa, sb query-command: "exact ARNOLD" + fields:sa, sb query-command: dynteaser } fieldset exactexplicit2 { - fields:sc, sd match { exact exact-terminator: "Arnold" } + fields:sc, sd } fieldset gram { diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 4a081edbf54..05a980a8347 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 13f59d4925f..9f01b6c45ce 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 8901acf63d1..28cb551cced 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 665edcdf45d..caae49a0252 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 7f9592dafc8..a045a532965 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -229,3 +259,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg new file mode 100644 index 00000000000..817640a89fd --- /dev/null +++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg @@ -0,0 +1,54 @@ +queryprofile[].id "default" +queryprofile[].type "DefaultQueryProfileType" +queryprofiletype[].id "DefaultQueryProfileType" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "native" +queryprofiletype[].field[].name "ranking" +queryprofiletype[].field[].type "query-profile:ranking_0_0" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].id "ranking_0_0" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "ranking" +queryprofiletype[].field[].name "features" +queryprofiletype[].field[].type "query-profile:features_0_1" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "rankfeature" +queryprofiletype[].id "features_0_1" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].field[].name "query(W_0)" +queryprofiletype[].field[].type "tensor(hidden[9],x[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_1)" +queryprofiletype[].field[].type "tensor(hidden[9],out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_out)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_0)" +queryprofiletype[].field[].type "tensor(hidden[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_1)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_out)" +queryprofiletype[].field[].type "tensor(out[1])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +enableGroupingSessionCache true diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml index e74152638fb..42336098a9a 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml +++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml @@ -1,5 +1,5 @@ <!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<query-profile-type id="DefaultQueryProfileType"> +<query-profile-type id="DefaultQueryProfileType" inherits="native"> <field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" /> <field name="ranking.features.query(b_0)" type="tensor(hidden[9])" /> <field name="ranking.features.query(W_1)" type="tensor(hidden[9],out[9])" /> diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index b33c2fbdf9b..f728b1b1d33 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,3 +235,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 47e07e2a524..458ee86031d 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 200 attribute[].densepostinglistthreshold 0.2 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index d7922a0de69..33fcecb1008 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 12dbf896edc..58125f73f9c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index e88dfde03bb..3404d6a0384 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 4634e120a3a..a8531f73c1e 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[2],y[1])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x{})" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index e6ffc37e871..290dd5b9a8b 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -271,3 +307,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/integration/lightgbm/models/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7b4b650295c..c010b23e207 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1315,7 +1315,6 @@ public class ModelProvisioningTest { } @Test - @Ignore // TODO: Enable when turning the port check on public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() { try { String services = diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java new file mode 100644 index 00000000000..2c13427760f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 8378ec811a5..e46208c770d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to bar and add another field SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); @@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to non-existing document bar SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); fooDocument.addField(fooRefToBarField); @@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest { resolver.resolveReferences(fooDocument); } - private static void addAttributeAspect(SDField fooRefToBarField) { - fooRefToBarField.parseIndexingScript("{ attribute }"); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..fcbb89b5c42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + Search parentSearch = new Search(); + SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch); + var parentField = new SDField("their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSearch.addDocument(parentDocument); + + Search fooSearch = new Search(); + SDField fooRefToParent = new SDField( + "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + var fooImports = fooSearch.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); + fooSearch.addDocument(fooDocument); + + Search barSearch = new Search(); + SDField barRefToParent = new SDField( + "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + var barImports = barSearch.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType("bar", barSearch); + barSearch.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index d67df3a5239..8ea53172200 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -10,6 +10,8 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import java.io.File; import java.io.IOException; @@ -53,6 +55,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase String path = exportConfig(name, config); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); + DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); return config; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 61065cd4bcc..e785792839d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -120,6 +120,11 @@ public class ExportingTestCase extends AbstractExportingTestCase { } @Test + public void testFieldSet2() throws IOException, ParseException { + assertCorrectDeriving("fieldset2"); + } + + @Test public void testIndexinfoFieldsets() throws IOException, ParseException { assertCorrectDeriving("indexinfo_fieldsets"); } @@ -150,4 +155,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectConfigFiles("tensor2"); } + @Test + public void testHnswIndex() throws IOException, ParseException { + assertCorrectDeriving("hnsw_index"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java index b299c7fa299..a6171901d2d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java @@ -1,16 +1,34 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.io.IOException; +import com.yahoo.component.ComponentId; + +import static org.junit.Assert.assertEquals; + public class NeuralNetTestCase extends AbstractExportingTestCase { @Test public void testNeuralNet() throws IOException, ParseException { - assertCorrectDeriving("neuralnet"); + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("neuralnet"); + + // Verify that query profiles end up correct when passed through the same intermediate forms as a full system + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]", + queryProfiles.getComponent("default")); + assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", + q.properties().get("ranking.features.query(b_1)").toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 08dd5148b29..0cd6674751e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import java.util.HashMap; @@ -33,6 +34,7 @@ class RankProfileSearchFixture { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); private final QueryProfileRegistry queryProfileRegistry; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index f0a0e389086..a306e0f2c90 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -1,8 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.collections.Pair; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.SearchBuilder; @@ -21,7 +23,6 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -274,11 +275,48 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true, logger); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } + @Test + public void noWarningWhenUsingTensorsWhenQueryFeaturesAreDeclared() throws Exception { + InspectableDeployLogger logger = new InspectableDeployLogger(); + SearchBuilder builder = new SearchBuilder(); + QueryProfileType myType = new QueryProfileType("mytype"); + myType.addField(new FieldDescription("rank.feature.query(foo)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(bar)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + myType.addField(new FieldDescription("rank.feature.query(baz)", + new TensorFieldType(TensorType.fromSpec("tensor(d[2])"))), + builder.getQueryProfileRegistry().getTypeRegistry()); + builder.getQueryProfileRegistry().getTypeRegistry().register(myType); + builder.importString(joinLines( + "search test {", + " document test { ", + " field anyfield type tensor(d[2]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " first-phase {", + " expression: sum(query(foo) + f() + sum(attribute(anyfield)))", + " }", + " function f() {", + " expression: query(bar) + query(baz)", + " }", + " }", + "}" + ), logger); + builder.build(true, logger); + String message = logger.findMessage("The following query features"); + assertNull(message); + } + private Map<String, ReferenceNode> summaryFeatures(RankProfile profile) { return profile.getSummaryFeatures().stream().collect(Collectors.toMap(f -> f.toString(), f -> f)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..79d19371f1c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index b6569357495..b9702c6c4f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; + +import static com.yahoo.searchdefinition.SearchBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -16,7 +21,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + createFromString(getSd("field f1 type array<tensor(x{})> {}")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -28,11 +33,12 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.", + assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", e.getMessage()); } } @@ -40,7 +46,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -51,7 +57,7 @@ public class TensorFieldTestCase { @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -59,8 +65,67 @@ public class TensorFieldTestCase { } } + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }")) + .getSearch().getAttribute("t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + private static String getSd(String field) { - return "search test {\n document test {\n" + field + "}\n}\n"; + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + System.out.println(sd); + var search = createFromString(sd).getSearch(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); } private void assertStartsWith(String prefix, String string) { diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java new file mode 100644 index 00000000000..d0ee0523489 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; + +/** + * Utility functions for testing generated configs for reference/imported fields. + */ +public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase { + + private static String TEST_FOLDER = "src/test/configmodel/types/references/"; + + protected void assertDocumentConfigs(DocumentModel model, + String cfgFileSpec) throws IOException { + assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); + assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); + } + + protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + + protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java new file mode 100644 index 00000000000..599ae77a456 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase { + + @Test + public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException { + assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " field person_ref type reference<person> { indexing: attribute }", + " }", + " import field campaign_ref.cool_field as my_cool_field {}", + " import field campaign_ref.swag_field as my_swag_field {}", + " import field person_ref.name as my_name {}", + "}")), + "multiple_imported_fields"); + } + + private static class TestDocumentModelBuilder { + private final SearchBuilder builder = new SearchBuilder(); + public TestDocumentModelBuilder addCampaign() throws ParseException { + builder.importString(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); + return this; + } + public TestDocumentModelBuilder addPerson() throws ParseException { + builder.importString(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + return this; + } + public DocumentModel build(String adSdContent) throws ParseException { + builder.importString(adSdContent); + builder.build(); + return builder.getModel(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index e9a7a6ed33e..55980ee5fea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -1,15 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; import java.io.IOException; @@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals; /** * @author geirst */ -public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase { @Test public void reference_fields_can_reference_other_document_types() throws ParseException, IOException { @@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT assertEquals(campaignRefType.getTargetType(), campaignType); } - private static String TEST_FOLDER = "src/test/configmodel/types/references/"; - - private void assertDocumentConfigs(DocumentModel model, - String cfgFileSpec) throws IOException { - assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); - assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); - } - - private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { - DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); - assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); - } - - private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { - DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); - assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); - } - private static class TestDocumentModelBuilder { private final SearchBuilder builder = new SearchBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 6fe69ac5c64..9265e4437f1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -12,6 +12,7 @@ import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Zone; @@ -66,6 +67,7 @@ import static org.junit.Assert.assertTrue; */ public class MetricsProxyContainerClusterTest { + private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); private static int numSystemMetrics = systemMetricSet.getMetrics().size(); @@ -237,6 +239,25 @@ public class MetricsProxyContainerClusterTest { } @Test + public void non_existent_metric_set_causes_exception() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-non-existent-default-set'>", + " <metric-set id='non-existent'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("No such metric-set: non-existent"); + consumersConfigFromXml(services, self_hosted); + } + + @Test public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { String services = String.join("\n", "<services>", @@ -262,6 +283,29 @@ public class MetricsProxyContainerClusterTest { } @Test + public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-public-default-set'>", + " <metric-set id='public'/>", + " <metric id='custom.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("custom.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + + @Test public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { String services = String.join("\n", "<services>", @@ -289,11 +333,11 @@ public class MetricsProxyContainerClusterTest { ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); assertEquals(Zone.defaultZone().system().value(), config.dimensions(AppDimensionNames.SYSTEM)); - assertEquals(zoneString(Zone.defaultZone()), config.dimensions(AppDimensionNames.ZONE)); + assertEquals(zoneString(Zone.defaultZone()), config.dimensions(PublicDimensions.ZONE)); assertEquals(MY_TENANT, config.dimensions(AppDimensionNames.TENANT)); assertEquals(MY_APPLICATION, config.dimensions(AppDimensionNames.APPLICATION)); assertEquals(MY_INSTANCE, config.dimensions(AppDimensionNames.INSTANCE)); - assertEquals(MY_TENANT + "." + MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.APPLICATION_ID)); + assertEquals(MY_TENANT + "." + MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(PublicDimensions.APPLICATION_ID)); assertEquals(MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.LEGACY_APPLICATION)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 89248d2469d..eddad6fce89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -1,19 +1,20 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames; import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; @@ -90,13 +91,28 @@ public class MetricsProxyContainerTest { String services = servicesWithContent(); VespaModel hostedModel = getModel(services, hosted); assertEquals(1, hostedModel.getHosts().size()); - String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname(); + String configId = containerConfigId(hostedModel, hosted); NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId); - assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); - assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); + assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE)); + assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID)); } + @Test + public void metrics_v2_handler_is_set_up_with_node_info_config() { + String services = servicesWithContent(); + VespaModel hostedModel = getModel(services, hosted); + + var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted)); + var handlers = container.getHandlers().getComponents(); + + assertEquals(1, handlers.size()); + var metricsV2Handler = handlers.iterator().next(); + + NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId()); + assertTrue(config.role().startsWith("content/my-content/0/")); + assertTrue(config.hostname().startsWith("node-1-3-9-")); + } @Test public void vespa_services_config_has_all_services() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index f3140aafdaf..7cbc9db5eb2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -49,7 +49,7 @@ class MetricsProxyModelTester { return tester.createModel(servicesXml, true); } - static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) { + static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) { return (mode == hosted) ? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname() : CONTAINER_CONFIG_ID; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index cdb4ce955e2..21df39ebde8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; /** * @author andreer */ -public class TlsSecretsValidatorTest { +public class EndpointCertificateSecretsValidatorTest { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @@ -43,21 +43,21 @@ public class TlsSecretsValidatorTest { @Test public void missing_certificate_fails_validation() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(TlsSecrets.MISSING)); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(EndpointCertificateSecrets.MISSING)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test public void validation_succeeds_with_certificate() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new TlsSecrets("cert", "key"))); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new EndpointCertificateSecrets("cert", "key"))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test @@ -65,10 +65,10 @@ public class TlsSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.empty()); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } - private static DeployState deployState(String servicesXml, String deploymentXml, Optional<TlsSecrets> tlsSecrets) { + private static DeployState deployState(String servicesXml, String deploymentXml, Optional<EndpointCertificateSecrets> endpointCertificateSecretsSecrets) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .withDeploymentSpec(deploymentXml) @@ -79,7 +79,7 @@ public class TlsSecretsValidatorTest { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(tlsSecrets)); + .setEndpointCertificateSecrets(endpointCertificateSecretsSecrets)); final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4064e53dfb7..3dfcef70aba 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest { headerfields, new StructDataType("bodyfields"), new FieldSets(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 4ea10c9a1c1..00ab175f496 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -18,30 +18,22 @@ import static org.junit.Assert.assertEquals; /** * @author gjoranv - * @since 5.5 */ public class AccessLogTest extends ContainerModelBuilderTestBase { @Test - public void default_access_log_is_only_added_when_search_is_present() { + public void default_access_log_is_added_by_default() { Element cluster1Elem = DomBuilderTest.parse( "<container id='cluster1' version='1.0'>", - "<search />", - nodesXml, - "</container>"); - Element cluster2Elem = DomBuilderTest.parse( - "<container id='cluster2' version='1.0'>", " <nodes>", " <node hostalias='mockhost' baseport='1234' />", " </nodes>", "</container>" ); - createModel(root, cluster1Elem, cluster2Elem); + createModel(root, cluster1Elem); assertNotNull(getJsonAccessLog("cluster1")); - assertNull( getJsonAccessLog("cluster2")); assertNull(getVespaAccessLog("cluster1")); - assertNull(getVespaAccessLog("cluster2")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 54d1c1c9793..53c99d1d3dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -25,6 +25,7 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.VipStatusHandler; +import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.servlet.ServletConfigConfig; @@ -39,7 +40,6 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainer; -import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; @@ -71,7 +71,9 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.isEmptyString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -149,7 +151,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - @Ignore // TODO: Enable when turning the port check on public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception { try { String servicesXml = @@ -224,12 +225,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertThat(defaultRootHandler.serverBindings(), contains("http://*/")); JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); - assertThat(applicationStatusHandler.serverBindings(), - contains("http://*/ApplicationStatus")); + assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus")); JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); - assertThat(fileRequestHandler.serverBindings(), - contains("http://*/status.html")); + assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html")); + + JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName()); + assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*")); } @Test @@ -693,7 +695,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) .build(); createModel(root, state, null, clusterElem); @@ -772,13 +774,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatProvidingTlsSecretOpensPort4443() { + public void requireThatProvidingEndpointCertificateSecretsOpensPort4443() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", nodesXml, "</container>" ); - DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build(); + DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build(); createModel(root, state, null, clusterElem); ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); @@ -801,6 +803,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("CERT", connectorConfig.ssl().certificate()); assertEquals("KEY", connectorConfig.ssl().privateKey()); assertEquals(4443, connectorConfig.listenPort()); + + assertThat("Connector must use Athenz truststore in a non-public system.", + connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem")); + assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 863781073f8..0f9bd506310 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas " <client-authentication>need</client-authentication>", " </ssl>", " </server>", + " <server port='9003' id='with-ciphers-and-protocols'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>", + " <protocols>TLSv1.3</protocols>", + " </ssl>", + " </server>", " </http>", nodesXml, "", @@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo(""))); assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH))); + ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols"); + assertTrue(withCiphersAndProtocols.ssl().enabled()); + assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384")))); + assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3")))); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS)); @@ -258,7 +273,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .modelHostProvisioner(new HostsXmlProvisioner(new StringReader(hostsxml))) .build(); MockRoot root = new MockRoot("root", deployState); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 21c384dfc69..883d8b89765 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -112,11 +112,7 @@ public class StorageClusterTest { "<cluster id=\"bees\">\n" + " <documents/>" + " <tuning>\n" + - " <persistence-threads>\n" + - " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + - " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + - " <thread count=\"1\"/>\n" + - " </persistence-threads>\n" + + " <persistence-threads count=\"7\"/>\n" + " </tuning>\n" + " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + @@ -130,6 +126,44 @@ public class StorageClusterTest { stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + assertEquals(false, config.enable_multibit_split_optimalization()); + } + { + assertEquals(1, stc.getChildren().size()); + StorageNode sn = stc.getChildren().values().iterator().next(); + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + sn.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + } + } + + @Test + public void testPersistenceThreadsOld() throws Exception { + + StorageCluster stc = parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>\n" + + " <persistence-threads>\n" + + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + + " <thread count=\"1\"/>\n" + + " </persistence-threads>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>", + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) + ); + + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + stc.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(4, config.num_threads()); assertEquals(false, config.enable_multibit_split_optimalization()); } @@ -161,7 +195,7 @@ public class StorageClusterTest { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(6, config.num_threads()); + assertEquals(8, config.num_threads()); } { assertEquals(1, stc.getChildren().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index ce36ecc4a1c..7a3b76db7f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; @@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -35,6 +36,7 @@ public class ImportedModelTester { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter(), new VespaImporter()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index c5c475360a3..ced7243adf5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -45,7 +45,7 @@ public class MlModelsTest { private void verify(VespaModel model) { assertEquals("Global models are created (although not used directly here", - 4, model.rankProfileList().getRankProfiles().size()); + 5, model.rankProfileList().getRankProfiles().size()); RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); model.getSearchClusters().get(0).getConfig(builder); @@ -71,8 +71,9 @@ public class MlModelsTest { "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + + "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" + "vespa.rank.firstphase: rankingExpression(firstphase)\n" + - "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" + + "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" + "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 3d4ac7f2eeb..2d3ddc33afb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -96,9 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); - assertEquals(4, config.rankprofile().size()); + assertEquals(5, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); + assertTrue(modelNames.contains("lightgbm_regression")); assertTrue(modelNames.contains("mnist_saved")); assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); @@ -112,13 +113,18 @@ public class ModelEvaluationTest { ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig)); - assertEquals(4, evaluator.models().size()); + assertEquals(5, evaluator.models().size()); Model xgboost = evaluator.models().get("xgboost_2_2"); assertNotNull(xgboost); assertNotNull(xgboost.evaluatorOf()); assertNotNull(xgboost.evaluatorOf("xgboost_2_2")); + Model lightgbm = evaluator.models().get("lightgbm_regression"); + assertNotNull(lightgbm); + assertNotNull(lightgbm.evaluatorOf()); + assertNotNull(lightgbm.evaluatorOf("lightgbm_regression")); + Model tensorflow_mnist = evaluator.models().get("mnist_saved"); assertNotNull(tensorflow_mnist); assertEquals(1, tensorflow_mnist.functions().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java index 1c4e005cb67..70e307e1748 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java @@ -18,12 +18,15 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and @@ -160,23 +163,30 @@ public class SearchClusterTest { AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex); assertEquals("xbulk", searchCluster2.getClusterName()); - Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal")); - assertNotNull(normalDispatcher); - assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId()); - DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal"); - assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host()); - - Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk")); - assertNotNull(xbulkDispatcher); - assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId()); - DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk"); - assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host()); + verifyDispatch(model, containerCluster1, "normal", "node2host"); + verifyDispatch(model, containerCluster1, "xbulk", "node0host"); + } + + private void verifyDispatch(VespaModel model, ContainerCluster containerCluster, String cluster, String host) { + Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster)); + assertNotNull(dispatcher); + assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId()); + DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); + + assertTrue(dispatcher.getInjectedComponentIds().contains("rpcresourcepool." + cluster)); + + Component<?,?> rpcResourcePool = (Component<?, ?>)dispatcher.getChildren().get("rpcresourcepool." + cluster); + assertNotNull(rpcResourcePool); + assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId()); + dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); } } diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 2bbd98f72ac..1bf42650123 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -119,6 +119,13 @@ <certificate-file>/foo/cert</certificate-file> <ca-certificates-file>/foo/cacerts</ca-certificates-file> <client-authentication>want</client-authentication> + <cipher-suites> + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + </cipher-suites> + <protocols>TLSv1.2,TLSv1.3</protocols> </ssl> </server> <server port="4083" id="sslProvider"> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 413e277655a..f66bacbc383 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -5,14 +5,13 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.ArrayList; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java new file mode 100644 index 00000000000..892ac639198 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -0,0 +1,17 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.zone; + +/** + * The routing methods supported by a zone. + * + * @author mpolden + */ +public enum RoutingMethod { + + /** Routing happens through shared routing layer */ + shared, + + /** Routing happens through a dedicated layer 4 load balancer */ + exclusive, + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 46efe7a440d..5ef23c80eac 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; import com.yahoo.config.provision.CloudName; @@ -22,6 +22,9 @@ public interface ZoneFilter { /** Zones which support direct routing through exclusive load balancers. */ ZoneList directlyRouted(); + /** Zones where traffic is routed using given method */ + ZoneList routingMethod(RoutingMethod method); + /** Zones where config servers are up and running. */ ZoneList reachable(); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index d809a3c97ed..ee843088086 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -69,7 +69,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) { Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>(); if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester - ret.put(ccs, JRTConfigRequester.get(new JRTConnectionPool(ccs), timingValues)); + ret.put(ccs, new JRTConfigRequester(new JRTConnectionPool(ccs), timingValues)); return ret; } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index c6f33a29410..7db9761d86a 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -35,7 +35,7 @@ class FileDistributionRpcServer { private final Supervisor supervisor; private final FileDownloader downloader; - private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("Rpc executor")); FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index a17f0abed92..3998d4f69d6 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -113,7 +113,7 @@ case $1 in if [ "$userargs" == "" ]; then userargs=$services__jvmargs_configproxy fi - jvmopts="-Xms32M -Xmx256M -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000000" + jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME @@ -122,6 +122,7 @@ case $1 in java ${jvmopts} \ -XX:+ExitOnOutOfMemoryError $(getJavaOptionsIPV46) \ -Dproxyconfigsources="${configsources}" ${userargs} \ + -XX:ActiveProcessorCount=2 \ -cp $cp com.yahoo.vespa.config.proxy.ProxyServer 19090 echo "Waiting for config proxy to start" diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 80c55c7b558..474c9f7a4db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -8,7 +8,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; -import java.util.*; +import java.util.ArrayList; +import java.util.List; /** * Deserializes config payload (cfg format) to a ConfigPayload. @@ -33,15 +34,10 @@ public class CfgConfigPayloadBuilder { int lineNum = 1; ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(); for (String line : lines) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "line " + lineNum + ": '" + line + "'"); - } parseLine(line, lineNum, payloadBuilder); lineNum++; } - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "payload=" + payloadBuilder.toString()); - } + log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -52,16 +48,13 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(LogLevel.DEBUG, "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); value=value.trim(); validateField(field, trimmedLine, lineNum); validateValue(value, trimmedLine, lineNum); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "field=" + field + ",value=" + value); - } List<String> fields = parseFieldList(field); ConfigPayloadBuilder currentBuilder = payloadBuilder; for (int fieldNum = 0; fieldNum < fields.size(); fieldNum++) { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java index 1069f1cdd53..7c042656bda 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -9,7 +9,6 @@ import com.yahoo.slime.Slime; * Implements a config instance serializer, serializing a config instance to a slime object. * * @author Ulf Lilleengen - * @since 5.1.14 */ public class ConfigInstanceSerializer implements Serializer { private final Slime slime; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java index fb2a3acbfdc..9710ee607eb 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -38,9 +38,7 @@ public class ConfigInstanceUtil { } } - public static <T extends ConfigInstance> T getNewInstance(Class<T> type, - String configId, - ConfigPayload payload) { + public static <T extends ConfigInstance> T getNewInstance(Class<T> type, String configId, ConfigPayload payload) { T instance; try { ConfigTransformer<?> transformer = new ConfigTransformer<>(type); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java index 1eaada5ae9d..121dba1e555 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -4,7 +4,6 @@ package com.yahoo.config.subscription; /** * This exception is thrown when any blocking call within the Config API is interrupted. * @author Ulf Lilleengen - * @since 5.1 */ @SuppressWarnings("serial") public class ConfigInterruptedException extends RuntimeException { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java index 57f4ad863f2..c3ceb81d3db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.config.ConfigKey; /** * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s * - * @author vegardh + * @author Vegard Havdal */ public class ConfigSet implements ConfigSource { private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java index d0eda9d27cc..0aac060abf6 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -4,7 +4,7 @@ package com.yahoo.config.subscription; /** * A type of source of config * - * @author vegardh + * @author Vegard Havdal */ public interface ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index ae6f14ad59e..2873855d1c2 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -25,7 +25,7 @@ import static java.util.stream.Collectors.toList; * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. * - * @author vegardh + * @author Vegard Havdal */ public class ConfigSubscriber implements AutoCloseable { @@ -325,7 +325,7 @@ public class ConfigSubscriber implements AutoCloseable { h.subscription().close(); } closeRequesters(); - log.log(LogLevel.DEBUG, "Config subscriber has been closed."); + log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java index d90a0493838..6e12a38faa3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -5,8 +5,7 @@ import java.io.File; /** * Source specifying config from a local directory - * @author vegardh - * @since 5.1 + * @author Vegard Havdal * */ public class DirSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java index 27129853eae..9b6fb6442e8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -6,7 +6,7 @@ import java.io.File; /** * Source specifying config from one local file * - * @author vegardh + * @author Vegard Havdal */ public class FileSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java index 6275befe18c..a098dec7f21 100644 --- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -5,8 +5,7 @@ import java.util.jar.JarFile; /** * Source specifying config as a jar file entry - * @author vegardh - * @since 5.1 + * @author Vegard Havdal * */ public class JarSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java index d68b503eb74..099a6bdc334 100644 --- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -4,7 +4,7 @@ package com.yahoo.config.subscription; /** * Source specifying raw config, where payload is given programmatically * - * @author vegardh + * @author Vegard Havdal */ public class RawSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java index aa80cc75ef0..d3a1bfb50a9 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -11,8 +11,7 @@ import java.lang.reflect.Constructor; /** * Subscription on a programmatically built set of configs - * @author vegardh - * @since 5.1 + * @author Vegard Havdal */ public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index c02301a0c17..3bf6093e872 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.config.protocol.DefContent; /** * Represents one active subscription to one config * - * @author vegardh + * @author Vegard Havdal */ public abstract class ConfigSubscription<T extends ConfigInstance> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index bcee06cd667..60c920d1dfa 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -18,7 +18,7 @@ import com.yahoo.log.LogLevel; /** * Subscription used when config id is file:... * - * @author vegardh + * @author Vegard Havdal */ public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java index bf247dffb7c..a40da85c88b 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.config.RawConfig; /** * A config handle which does not use the config class, but payload instead. Used in config proxy. * - * @author vegardh + * @author Vegard Havdal */ @SuppressWarnings({"rawtypes", "unchecked"}) public class GenericConfigHandle extends ConfigHandle { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java index 82d4708f53b..8c0a8f27555 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.config.TimingValues; /** * A subscriber that can subscribe without the class. Used by configproxy. * - * @author vegardh + * @author Vegard Havdal */ public class GenericConfigSubscriber extends ConfigSubscriber { @@ -24,7 +24,7 @@ public class GenericConfigSubscriber extends ConfigSubscriber { * Constructs a new subscriber using the given pool of requesters (JRTConfigRequester holds 1 connection which in * turn is subject to failover across the elems in the source set.) * The behaviour is undefined if the map key is different from the source set the requester was built with. - * See also {@link JRTConfigRequester#get(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} + * See also {@link JRTConfigRequester#JRTConfigRequester(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} * * @param requesters a map from config source set to config requester */ @@ -32,10 +32,6 @@ public class GenericConfigSubscriber extends ConfigSubscriber { this.requesters = requesters; } - public GenericConfigSubscriber() { - super(); - } - /** * Subscribes to config without using the class. For internal use in config proxy. * diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index c1f9ce02650..8ce3449fba5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. * - * @author vegardh + * @author Vegard Havdal * */ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfig> { @@ -33,9 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi @Override protected void setNewConfig(JRTClientConfigRequest jrtReq) { setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsInternalRedeploy(), RawConfig.createFromResponseParameters(jrtReq) ); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "in setNewConfig, config=" + this.getConfigState().getConfig()); - } + log.log(LogLevel.DEBUG, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 06c772ac456..6d08976b61c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -52,20 +52,11 @@ public class JRTConfigRequester implements RequestWaiter { /** * Returns a new requester - * @param connectionPool The connectionPool to use - * @param timingValues The timing values - * @return new requester object - */ - public static JRTConfigRequester get(ConnectionPool connectionPool, TimingValues timingValues) { - return new JRTConfigRequester(connectionPool, timingValues); - } - - /** - * New requester - * @param connectionPool the connectionPool this requester should use - * @param timingValues timeouts and delays used when sending JRT config requests + * + * @param connectionPool the connectionPool this requester should use + * @param timingValues timeouts and delays used when sending JRT config requests */ - JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { this.connectionPool = connectionPool; this.timingValues = timingValues; } @@ -86,11 +77,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Requesting config for " + sub + " on connection " + connection - + " with client timeout " + jrtClientTimeout + - (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); - } + log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + + " with client timeout " + jrtClientTimeout + + (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -120,7 +109,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(LogLevel.SPAM, trace::toString); + log.log(LogLevel.SPAM, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -157,7 +146,7 @@ public class JRTConfigRequester implements RequestWaiter { // The subscription object has an "old" config, which is all we have to offer back now log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } - final ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); + ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); long delay = calculateFailedRequestDelay(errorType, transientFailures, fatalFailures, timingValues, configured); if (errorType == ErrorType.TRANSIENT) { @@ -233,7 +222,8 @@ public class JRTConfigRequester implements RequestWaiter { suspendWarningLogged = Instant.MIN; noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); - sub.setLastCallBackOKTS(System.currentTimeMillis()); + sub.setLastCallBackOKTS(Instant.now()); + log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); @@ -257,7 +247,7 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(LogLevel.DEBUG, timingValues::toString); + log.log(LogLevel.SPAM, () -> timingValues.toString()); log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index d9366c28b9b..39e6c69f539 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -28,8 +30,8 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester requester; private TimingValues timingValues; - // Last time we got an OK JRT callback for this - private long lastOK = 0; + // Last time we got an OK JRT callback + private Instant lastOK = Instant.MIN; /** * The queue containing either nothing or the one (newest) request that has got callback from JRT, @@ -40,9 +42,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { super(key, subscriber); - this.timingValues=timingValues; + this.timingValues = timingValues; if (source instanceof ConfigSourceSet) { - this.sources=(ConfigSourceSet) source; + this.sources = (ConfigSourceSet) source; } } @@ -53,7 +55,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc ConfigState<T> configState = getConfigState(); boolean gotNew = configState.isGenerationChanged() || configState.isConfigChanged() || hasException(); // Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check - if (getReqQueue().peek()==null && gotNew) { + if (getReqQueue().peek() == null && gotNew) { return true; } // Otherwise poll the queue for another generation or timeout @@ -88,6 +90,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // timed out, we know nothing new. return false; } + log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -135,11 +138,11 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc @Override public boolean subscribe(long timeout) { - lastOK=System.currentTimeMillis(); + lastOK = Instant.now(); requester = getRequester(); requester.request(this); JRTClientConfigRequest req = reqQueue.peek(); - while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) { + while (req == null && (Instant.now().isBefore(lastOK.plus(Duration.ofMillis(timeout))))) { try { Thread.sleep(10); } catch (InterruptedException e) { @@ -152,7 +155,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester getRequester() { JRTConfigRequester requester = subscriber.requesters().get(sources); - if (requester==null) { + if (requester == null) { requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); subscriber.requesters().put(sources, requester); } @@ -164,8 +167,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc public void close() { super.close(); reqQueue = new LinkedBlockingQueue<>() { + @SuppressWarnings("NullableProblems") @Override - public void put(JRTClientConfigRequest e) throws InterruptedException { + public void put(JRTClientConfigRequest e) { // When closed, throw away all requests that callbacks try to put } }; @@ -191,7 +195,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); } - void setLastCallBackOKTS(long lastCallBackOKTS) { + void setLastCallBackOKTS(Instant lastCallBackOKTS) { this.lastOK = lastCallBackOKTS; } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index a96499482c5..05da9a72837 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.config.ConfigPayload; /** * Subscription to use when config id is jar:.../foo.jar[!/pathInJar/] * - * @author vegardh + * @author Vegard Havdal * @author gjoranv */ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { @@ -33,8 +33,8 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // jar:configs/app.jar!/configs/ JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) { super(key, subscriber); - this.jarName=jarName; - this.path=path; + this.jarName = jarName; + this.path = path; } @Override @@ -43,17 +43,18 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // Not supporting changing the payload for jar return true; } - if (zipEntry==null) { + if (zipEntry == null) { // First time polled - JarFile jarFile = null; + JarFile jarFile; try { jarFile = new JarFile(jarName); } catch (IOException e) { throw new IllegalArgumentException(e); } zipEntry = getEntry(jarFile, path); - if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); - T config = null; + if (zipEntry == null) + throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); + T config; try { ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n"))); config = payload.toInstance(configClass, key.getConfigId()); @@ -78,6 +79,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } return false; } + /** * Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in * the given JarFile. diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index 9ad8f5c6ba2..58eed7f9e78 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.util.ConfigUtils; * For unit testing * * @author hmusum - * @since 5.1.11 */ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index bc546cfdd1f..68ff6bb0135 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -12,10 +12,10 @@ import com.yahoo.vespa.config.ConfigPayload; /** * Subscription used when config id is raw:... - * + * <p> * Config is the actual text given after the config id, with newlines * - * @author vegardh + * @author Vegard Havdal */ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { @@ -24,7 +24,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) { super(key, subscriber); - this.inputPayload=pl; + this.inputPayload = pl; } @Override @@ -32,7 +32,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc if (checkReloaded()) { return true; } - if (payload==null) { + if (payload == null) { payload = inputPayload; ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId())); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java index 55cb05ec230..6fcdc19cf97 100755 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -78,7 +78,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable if (!(o instanceof ConfigKey)) { return false; } - ConfigKey<?> key = (ConfigKey) o; + ConfigKey<?> key = (ConfigKey<?>) o; return (name.equals(key.name) && configId.equals(key.configId) && namespace.equals(key.namespace)); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java index 4f02df04b9b..f4858843574 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java @@ -20,7 +20,6 @@ import java.io.OutputStream; * A class that holds a representation of a config payload. * * @author Ulf Lilleengen - * @since 5.1.6 */ public class ConfigPayload { private final Slime slime; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 65106f158fc..134352736b6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -33,7 +33,6 @@ import java.util.logging.Logger; * TODO: This can be refactored a lot, since many of the reflection methods are duplicated * * @author Ulf Lilleengen, hmusum, Tony Vaagenes - * @since 5.1.6 */ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName()); @@ -480,15 +479,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } private void debug(String message) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, message); - } + log.log(LogLevel.DEBUG, () -> message); } private void trace(String message) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, message); - } + log.log(LogLevel.SPAM, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java index bb974ddae42..2e470ec55e4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java @@ -12,7 +12,6 @@ import java.util.*; * * TODO: Add toString * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigPayloadBuilder { diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index 163e010cdd6..6d2ae3ef13e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -12,7 +12,6 @@ import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer; * A utility class that can be used to transform config from one format to another. * * @author Ulf Lilleengen, hmusum, Tony Vaagenes - * @since 5.1.6 */ public class ConfigTransformer<T extends ConfigInstance> { /** diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java index f34b66e17a8..5bbc985cce0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java @@ -11,7 +11,6 @@ import com.yahoo.slime.*; * TODO: Support giving correct type of default values * * @author Ulf Lilleengen - * @since 5.1 */ public class DefaultValueApplier { @@ -31,21 +30,11 @@ public class DefaultValueApplier { } private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) { - cursor.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - applyDefaultsToObject(cursor.field(name), def); - } - }); + cursor.traverse((ObjectTraverser) (name, inspector) -> applyDefaultsToObject(cursor.field(name), def)); } private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) { - cursor.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - applyDefaultsToObject(cursor.entry(idx), def); - } - }); + cursor.traverse((ArrayTraverser) (idx, inspector) -> applyDefaultsToObject(cursor.entry(idx), def)); } private void applyDefaultsToObject(Cursor cursor, InnerCNode def) { diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java index c151121f0e9..3da09d07016 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java +++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java @@ -5,7 +5,6 @@ package com.yahoo.vespa.config; * Interface for counters. * * @author Ulf Lilleengen - * @since 5.9 */ public interface GenerationCounter { /** diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java index 2fbbda2aa38..2f351cc2bd4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigInstance; /** * - /** * A generic config with an internal generic builder that mimics a real config builder in order to support builders * when we don't have the schema. * diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index efeaacf225b..a4da996effd 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -74,18 +74,14 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List<JRTConnection> sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Choosing new connection: " + currentConnection); - } + log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); return currentConnection; } List<JRTConnection> getSources() { - List<JRTConnection> ret = new ArrayList<>(); + List<JRTConnection> ret; synchronized (connections) { - for (JRTConnection source : connections.values()) { - ret.add(source); - } + ret = new ArrayList<>(connections.values()); } return ret; } diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java index 113d561bf87..780cf657009 100644 --- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -38,7 +38,6 @@ public class TimingValues { long configuredErrorDelay, long fixedDelay, int maxDelayMultiplier) { - this.successTimeout = successTimeout; this.errorTimeout = errorTimeout; this.initialTimeout = initialTimeout; @@ -51,15 +50,14 @@ public class TimingValues { } private TimingValues(long successTimeout, - long errorTimeout, - long initialTimeout, - long subscribeTimeout, - long unconfiguredDelay, - long configuredErrorDelay, - long fixedDelay, - int maxDelayMultiplier, - Random rand) { - + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier, + Random rand) { this.successTimeout = successTimeout; this.errorTimeout = errorTimeout; this.initialTimeout = initialTimeout; @@ -71,18 +69,6 @@ public class TimingValues { this.rand = rand; } - public TimingValues(TimingValues tv) { - this(tv.successTimeout, - tv.errorTimeout, - tv.initialTimeout, - tv.subscribeTimeout, - tv.unconfiguredDelay, - tv.configuredErrorDelay, - tv.fixedDelay, - tv.maxDelayMultiplier, - tv.getRandom()); - } - public TimingValues(TimingValues tv, Random random) { this(tv.successTimeout, tv.errorTimeout, diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index b7d9f9e14ca..863e95c5625 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -47,7 +47,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -78,7 +78,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java index 7ac03ea6151..05125f8b08b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java @@ -11,7 +11,6 @@ import javax.tools.JavaFileObject; * TODO: Assumes that diagnostics is the same as given to the task, not ideal. * * @author Ulf Lilleengen - * @since 5.2 */ class CompilationTask { private final JavaCompiler.CompilationTask task; diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java index 80909ab653d..fa5cf427aad 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance; * Represents a builder that can be instantiated. * * @author Ulf Lilleengen - * @since 5.2 */ public interface CompiledBuilder { <BUILDER extends ConfigInstance.Builder> BUILDER newInstance(); diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java index 1939a007059..c67d2121844 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java @@ -8,7 +8,6 @@ import java.net.URI; * Represents an in memory source object that can be compiled. * * @author Ulf Lilleengen - * @since 5.2 */ class StringSourceObject extends SimpleJavaFileObject { private final String code; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java index ba9fa601c81..bc6f97a3e00 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java @@ -10,7 +10,6 @@ import java.io.IOException; * Contains info relevant for compression and decompression. * * @author Ulf Lilleengen - * @since 5.19 */ public class CompressionInfo { private static final String COMPRESSION_TYPE = "compressionType"; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java index c6548f63cd7..5fbd61cde3d 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.protocol; /** * @author Ulf Lilleengen - * @since 5.18 */ public enum CompressionType { UNCOMPRESSED, LZ4; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java index c07be8337fe..bb0ee2bb935 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -5,7 +5,6 @@ import com.yahoo.text.Utf8Array; import java.io.IOException; import java.io.OutputStream; -import java.util.List; /** * A config response encapsulates the payload and some meta information. This makes it possible to @@ -19,8 +18,6 @@ public interface ConfigResponse { Utf8Array getPayload(); - List<String> getLegacyPayload(); - long getGeneration(); boolean isInternalRedeploy(); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java index ea2cf1a1bd8..3aa1898fcd1 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java @@ -12,7 +12,6 @@ import java.util.List; /** * @author Ulf Lilleengen -* @since 5.3 */ public class DefContent { private final List<String> data; @@ -35,12 +34,7 @@ public class DefContent { static DefContent fromSlime(Inspector data) { final List<String> lst = new ArrayList<>(); - data.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - lst.add(inspector.asString()); - } - }); + data.traverse((ArrayTraverser) (idx, inspector) -> lst.add(inspector.asString())); return new DefContent(lst); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java index 25fecba425c..ab47fec0641 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -64,14 +64,6 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { String getNewConfigMd5(); /** - * Test whether or not the payload is contained in this response or not. - * Should return false for error responses as well. - * - * @return true if empty, false if not. - */ - boolean containsPayload(); - - /** * Test whether or not the response contains an updated config or not. * False if no response has been returned. * diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index 1c842a4d1b0..12e5968ab83 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -5,13 +5,20 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigSubscription; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.util.ConfigUtils; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.Optional; +import java.util.logging.Logger; /** * Represents version 3 config request for config clients. Provides methods for inspecting request and response @@ -21,7 +28,12 @@ import java.util.Optional; * * @author Ulf Lilleengen */ -public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { +public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { + + protected static final Logger log = Logger.getLogger(JRTClientConfigRequestV3.class.getName()); + protected final SlimeRequestData requestData; + protected final Request request; + private final SlimeResponseData responseData; protected JRTClientConfigRequestV3(ConfigKey<?> key, String hostname, @@ -32,15 +44,38 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { - super(key, hostname, defSchema, configMd5, generation, timeout, trace, compressionType, vespaVersion); + Slime data = SlimeRequestData.encodeRequest(key, + hostname, + defSchema, + configMd5, + generation, + timeout, + trace, + getProtocolVersion(), + compressionType, + vespaVersion); + Request jrtReq = new Request(getJRTMethodName()); + jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data))); + + this.requestData = new SlimeRequestData(jrtReq, data); + this.responseData = new SlimeResponseData(jrtReq); + this.request = jrtReq; + } + + protected static String encodeAsUtf8String(Slime data) { + ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + try { + new JsonFormat(true /* compact format */).encode(baos, data); + } catch (IOException e) { + throw new RuntimeException("Unable to encode config request", e); + } + return Utf8.toString(baos.toByteArray()); } - @Override protected String getJRTMethodName() { return JRTMethods.configV3getConfigMethodName; } - @Override protected boolean checkReturnTypes(Request request) { return JRTMethods.checkV3ReturnTypes(request); } @@ -74,22 +109,19 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { - String hostname = ConfigUtils.getCanonicalHostName(); - ConfigKey<T> key = sub.getKey(); ConfigSubscription.ConfigState<T> configState = sub.getConfigState(); - T i = configState.getConfig(); - return createWithParams(key, - sub.getDefContent(), - hostname, - i != null ? i.getConfigMd5() : "", - configState.getGeneration() != null ? configState.getGeneration() : 0L, - sub.timingValues().getSubscribeTimeout(), - trace, - compressionType, - vespaVersion); + T config = configState.getConfig(); + return createWithParams(sub.getKey(), + sub.getDefContent(), + ConfigUtils.getCanonicalHostName(), + config != null ? config.getConfigMd5() : "", + configState.getGeneration(), + sub.timingValues().getSubscribeTimeout(), + trace, + compressionType, + vespaVersion); } - public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout, Trace trace, @@ -107,7 +139,6 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { vespaVersion); } - public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey, DefContent defContent, String hostname, @@ -133,4 +164,142 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { return requestData.getVespaVersion(); } + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()) + .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) + .append("'\n"); + sb.append("response='").append(getNewConfigMd5()) + .append(",").append(getNewGeneration()) + .append(",").append(responseIsInternalRedeploy()) + .append("'\n"); + return sb.toString(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + @Override + public boolean hasUpdatedGeneration() { + long prevGen = getRequestGeneration(); + long newGen = getNewGeneration(); + return ConfigUtils.isGenerationNewer(newGen, prevGen); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + protected String newConfMd5() { + String newMd5 = getNewConfigMd5(); + if ("".equals(newMd5)) return getRequestConfigMd5(); + return newMd5; + } + + protected long newGen() { + long newGen = getNewGeneration(); + if (newGen == 0) return getRequestGeneration(); + return newGen; + } + + @Override + public DefContent getDefContent() { + return requestData.getSchema(); + } + + @Override + public boolean isError() { + return request.isError(); + } + + @Override + public boolean hasUpdatedConfig() { + String respMd5 = getNewConfigMd5(); + return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); + } + + @Override + public Trace getResponseTrace() { + return responseData.getResponseTrace(); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + @Override + public boolean validateResponse() { + if (request.isError()) { + return false; + } else if (request.returnValues().size() == 0) { + return false; + } else if (!checkReturnTypes(request)) { + log.warning("Invalid return types for config response: " + errorMessage()); + return false; + } + return true; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + return (errorCode == 0); + } + + @Override + public String getNewConfigMd5() { + return responseData.getResponseConfigMd5(); + } + + @Override + public long getNewGeneration() { + return responseData.getResponseConfigGeneration(); + } + + @Override + public boolean responseIsInternalRedeploy() { + return responseData.getResponseInternalRedeployment(); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + protected SlimeResponseData getResponseData() { + return responseData; + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java index 9dec28b13e0..0861c5008c0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java @@ -12,7 +12,6 @@ import java.util.*; * To hide JRT implementations. * * @author Ulf Lilleengen - * @since 5.3 */ public class JRTConfigRequestFactory { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index f70ebf39a28..3609ba04424 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -1,17 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; -import com.yahoo.log.LogLevel; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Value; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; /** - * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields JRT + * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields: * * * A metadata field containing json data describing config generation, md5 and compression info * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload @@ -22,14 +29,41 @@ import java.io.IOException; * * @author Ulf Lilleengen */ -// TODO: Merge with parent -public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { +public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { + protected static final Logger log = Logger.getLogger(JRTServerConfigRequestV3.class.getName()); + private static final JsonFactory jsonFactory = new JsonFactory(); + protected final Request request; + private final SlimeRequestData requestData; /** Response field */ private boolean internalRedeploy = false; + // Response values + private boolean isDelayed = false; + private Trace requestTrace = null; protected JRTServerConfigRequestV3(Request request) { - super(request); + this.requestData = new SlimeRequestData(request); + this.request = request; + } + + protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { + return jsonFactory.createGenerator(byteArrayOutputStream); + } + + protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { + return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { + jsonGenerator.writeStringField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { + jsonGenerator.writeNumberField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException { + jsonGenerator.writeBooleanField(fieldName, value); } @Override @@ -57,9 +91,7 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { } compressionInfo.serialize(jsonGenerator); jsonGenerator.writeEndObject(); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, getConfigKey() + ": response dataXXXXX" + payload.withCompression(CompressionType.UNCOMPRESSED) + "XXXXX"); - } + jsonGenerator.writeEndObject(); jsonGenerator.close(); } catch (IOException e) { @@ -85,4 +117,145 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { return new JRTServerConfigRequestV3(req); } + @Override + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public DefContent getDefContent() { + return getSchema(); + } + + @Override + public boolean noCache() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()).append("'\n"); + return sb.toString(); + } + + @Override + public Payload payloadFromResponse(ConfigResponse response) { + return Payload.from(response.getPayload(), response.getCompressionInfo()); + } + + private DefContent getSchema() { + return requestData.getSchema(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + public Trace getRequestTrace() { + if (requestTrace == null) { + requestTrace = requestData.getRequestTrace(); + } + return requestTrace; + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + if (errorCode != 0) { + addErrorResponse(errorCode); + } + return (errorCode == 0); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + private void addErrorResponse(int errorCode) { + addErrorResponse(errorCode, ErrorCode.getName(errorCode)); + } + + @Override + public void setDelayedResponse(boolean delayedResponse) { + this.isDelayed = delayedResponse; + } + + @Override + public void addErrorResponse(int errorCode, String name) { + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); + try { + JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); + jsonWriter.writeStartObject(); + addCommonReturnValues(jsonWriter); + jsonWriter.writeEndObject(); + jsonWriter.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add error response for " + this); + } + request.setError(errorCode, name); + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + } + + protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { + ConfigKey<?> key = requestData.getConfigKey(); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); + jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); + jsonGenerator.writeRawValue(getRequestTrace().toString(true)); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + @Override + public boolean isDelayedResponse() { + return isDelayed; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + protected CompressionType getCompressionType() { + return requestData.getCompressionType(); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java index 79746127bcd..61a742ade3b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java @@ -7,7 +7,6 @@ import java.io.ByteArrayOutputStream; * Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method. * * @author Ulf Lilleengen - * @since 5.19 */ class NoCopyByteArrayOutputStream extends ByteArrayOutputStream { public NoCopyByteArrayOutputStream() { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index fed9a96504d..8c7c7f2703e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -14,7 +14,6 @@ import java.util.regex.Pattern; * Static utility methods for verifying common request properties. * * @author Ulf Lilleengen - * @since 5.3 */ public class RequestValidation { private static final Logger log = Logger.getLogger(RequestValidation.class.getName()); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java deleted file mode 100644 index 3ccf71ec519..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.protocol; - -import com.yahoo.jrt.Request; -import com.yahoo.jrt.StringValue; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.util.ConfigUtils; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of - * payload encoding and decoding, as well as adding extra request/response fields. - * - * @author Ulf Lilleengen - */ -public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest { - - protected static final Logger log = Logger.getLogger(SlimeClientConfigRequest.class.getName()); - - protected final SlimeRequestData requestData; - private final SlimeResponseData responseData; - - protected final Request request; - - protected SlimeClientConfigRequest(ConfigKey<?> key, - String hostname, - DefContent defSchema, - String configMd5, - long generation, - long timeout, - Trace trace, - CompressionType compressionType, - Optional<VespaVersion> vespaVersion) { - Slime data = SlimeRequestData.encodeRequest(key, - hostname, - defSchema, - configMd5, - generation, - timeout, - trace, - getProtocolVersion(), - compressionType, - vespaVersion); - Request jrtReq = new Request(getJRTMethodName()); - jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true))); - - this.requestData = new SlimeRequestData(jrtReq, data); - this.responseData = new SlimeResponseData(jrtReq); - this.request = jrtReq; - } - - protected abstract String getJRTMethodName(); - - protected static String encodeAsUtf8String(Slime data, boolean compact) { - ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); - try { - new JsonFormat(compact).encode(baos, data); - } catch (IOException e) { - throw new RuntimeException("Unable to encode config request", e); - } - return Utf8.toString(baos.toByteArray()); - } - - public ConfigKey<?> getConfigKey() { - return requestData.getConfigKey(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("request='").append(getConfigKey()) - .append(",").append(getClientHostName()) - .append(",").append(getRequestConfigMd5()) - .append(",").append(getRequestGeneration()) - .append(",").append(getTimeout()) - .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) - .append("'\n"); - sb.append("response='").append(getNewConfigMd5()) - .append(",").append(getNewGeneration()) - .append(",").append(responseIsInternalRedeploy()) - .append("'\n"); - return sb.toString(); - } - - @Override - public String getClientHostName() { - return requestData.getClientHostName(); - } - - @Override - public Request getRequest() { - return request; - } - - @Override - public int errorCode() { - return request.errorCode(); - } - - @Override - public String errorMessage() { - return request.errorMessage(); - } - - @Override - public String getShortDescription() { - return toString(); - } - - @Override - public boolean hasUpdatedGeneration() { - long prevGen = getRequestGeneration(); - long newGen = getNewGeneration(); - return ConfigUtils.isGenerationNewer(newGen, prevGen); - } - - @Override - public long getTimeout() { - return requestData.getTimeout(); - } - - protected String newConfMd5() { - String newMd5 = getNewConfigMd5(); - if ("".equals(newMd5)) return getRequestConfigMd5(); - return newMd5; - } - - protected long newGen() { - long newGen = getNewGeneration(); - if (newGen==0) return getRequestGeneration(); - return newGen; - } - - @Override - public DefContent getDefContent() { - return requestData.getSchema(); - } - - @Override - public boolean isError() { - return request.isError(); - } - - @Override - public boolean containsPayload() { - return false; - } - - @Override - public boolean hasUpdatedConfig() { - String respMd5 = getNewConfigMd5(); - return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); - } - - @Override - public Trace getResponseTrace() { - return responseData.getResponseTrace(); - } - - @Override - public String getRequestConfigMd5() { - return requestData.getRequestConfigMd5(); - } - - @Override - public boolean validateResponse() { - if (request.isError()) { - return false; - } else if (request.returnValues().size() == 0) { - return false; - } else if (!checkReturnTypes(request)) { - log.warning("Invalid return types for config response: " + errorMessage()); - return false; - } - return true; - } - - @Override - public boolean validateParameters() { - int errorCode = RequestValidation.validateRequest(this); - return (errorCode == 0); - } - - protected abstract boolean checkReturnTypes(Request request); - - @Override - public String getNewConfigMd5() { - return responseData.getResponseConfigMd5(); - } - - @Override - public long getNewGeneration() { - return responseData.getResponseConfigGeneration(); - } - - @Override - public boolean responseIsInternalRedeploy() { - return responseData.getResponseInternalRedeployment(); - } - - @Override - public long getRequestGeneration() { - return requestData.getRequestGeneration(); - } - - protected SlimeResponseData getResponseData() { - return responseData; - } - - public Optional<VespaVersion> getVespaVersion() { - return requestData.getVespaVersion(); - } - -} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java index 327acab53d3..ff0b7f964bf 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -1,17 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.text.Utf8Array; -import com.yahoo.vespa.config.ConfigFileFormat; import com.yahoo.vespa.config.ConfigPayload; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; /** * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. @@ -22,27 +16,24 @@ public class SlimeConfigResponse implements ConfigResponse { private final Utf8Array payload; private final CompressionInfo compressionInfo; - private final InnerCNode targetDef; private final long generation; private final boolean internalRedeploy; private final String configMd5; - public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation, boolean internalRedeploy, String configMd5) { Utf8Array data = payload.toUtf8Array(true); - return new SlimeConfigResponse(data, targetDef, generation, internalRedeploy, + return new SlimeConfigResponse(data, generation, internalRedeploy, configMd5, CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); } public SlimeConfigResponse(Utf8Array payload, - InnerCNode targetDef, long generation, boolean internalRedeploy, String configMd5, CompressionInfo compressionInfo) { this.payload = payload; - this.targetDef = targetDef; this.generation = generation; this.internalRedeploy = internalRedeploy; this.configMd5 = configMd5; @@ -55,19 +46,6 @@ public class SlimeConfigResponse implements ConfigResponse { } @Override - public List<String> getLegacyPayload() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ConfigFileFormat format = new ConfigFileFormat(targetDef); - Payload v1payload = Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); - try { - ConfigPayload.fromUtf8Array(v1payload.getData()).serialize(baos, format); - return Arrays.asList(baos.toString(StandardCharsets.UTF_8).split("\\n")); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override public long getGeneration() { return generation; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java deleted file mode 100644 index 34d6f90cbcb..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.protocol; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.yahoo.jrt.*; -import com.yahoo.slime.*; -import com.yahoo.text.Utf8Array; -import com.yahoo.vespa.config.*; -import com.yahoo.vespa.config.ErrorCode; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of - * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3 - * config protocol. - * - * @author Ulf Lilleengen - */ -abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { - - protected static final Logger log = Logger.getLogger(SlimeServerConfigRequest.class.getName()); - - private static final JsonFactory jsonFactory = new JsonFactory(); - - private final SlimeRequestData requestData; - - // Response values - private boolean isDelayed = false; - private Trace requestTrace = null; - protected final Request request; - - protected SlimeServerConfigRequest(Request request) { - this.requestData = new SlimeRequestData(request); - this.request = request; - } - - protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { - return jsonFactory.createGenerator(byteArrayOutputStream); - } - - @Override - public ConfigKey<?> getConfigKey() { - return requestData.getConfigKey(); - } - - @Override - public DefContent getDefContent() { - return getSchema(); - } - - @Override - public boolean noCache() { - return false; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("request='").append(getConfigKey()) - .append(",").append(getClientHostName()) - .append(",").append(getRequestConfigMd5()) - .append(",").append(getRequestGeneration()) - .append(",").append(getTimeout()).append("'\n"); - return sb.toString(); - } - - @Override - public Payload payloadFromResponse(ConfigResponse response) { - return Payload.from(response.getPayload(), response.getCompressionInfo()); - } - - private DefContent getSchema() { - return requestData.getSchema(); - } - - @Override - public String getClientHostName() { - return requestData.getClientHostName(); - } - - public Trace getRequestTrace() { - if (requestTrace == null) { - requestTrace = requestData.getRequestTrace(); - } - return requestTrace; - } - - @Override - public Request getRequest() { - return request; - } - - @Override - public boolean validateParameters() { - int errorCode = RequestValidation.validateRequest(this); - if (errorCode != 0) { - addErrorResponse(errorCode); - } - return (errorCode == 0); - } - - @Override - public String getRequestConfigMd5() { - return requestData.getRequestConfigMd5(); - } - - private void addErrorResponse(int errorCode) { - addErrorResponse(errorCode, ErrorCode.getName(errorCode)); - } - - @Override - public void setDelayedResponse(boolean delayedResponse) { - this.isDelayed = delayedResponse; - } - - @Override - public void addErrorResponse(int errorCode, String name) { - ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); - try { - JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); - jsonWriter.writeStartObject(); - addCommonReturnValues(jsonWriter); - jsonWriter.writeEndObject(); - jsonWriter.close(); - } catch (IOException e) { - throw new IllegalArgumentException("Could not add error response for " + this); - } - request.setError(errorCode, name); - request.returnValues().add(createResponseValue(byteArrayOutputStream)); - } - - protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { - return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); - } - - protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { - ConfigKey<?> key = requestData.getConfigKey(); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); - jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); - jsonGenerator.writeRawValue(getRequestTrace().toString(true)); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { - jsonGenerator.writeStringField(fieldName, value); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { - jsonGenerator.writeNumberField(fieldName, value); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException { - jsonGenerator.writeBooleanField(fieldName, value); - } - - @Override - public long getRequestGeneration() { - return requestData.getRequestGeneration(); - } - - @Override - public boolean isDelayedResponse() { - return isDelayed; - } - - @Override - public int errorCode() { - return request.errorCode(); - } - - @Override - public String errorMessage() { - return request.errorMessage(); - } - - @Override - public String getShortDescription() { - return toString(); - } - - protected CompressionType getCompressionType() { - return requestData.getCompressionType(); - } - - @Override - public long getTimeout() { - return requestData.getTimeout(); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return requestData.getVespaVersion(); - } - -} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java index f4754f4b0e1..998d2c8bcf5 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java @@ -9,7 +9,6 @@ import com.yahoo.yolean.trace.TraceNode; * Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode} * * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceDeserializer { private final Inspector entry; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java index 62fa6f8a882..5f5f057a26b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -12,7 +12,6 @@ import java.util.Stack; * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. * * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceSerializer extends TraceVisitor { static final String TIMESTAMP = "timestamp"; @@ -30,7 +29,6 @@ public class SlimeTraceSerializer extends TraceVisitor { current.setLong(TIMESTAMP, node.timestamp()); encodePayload(current, node.payload()); addChildrenCursors(current, node); - } private void encodePayload(Cursor current, Object payload) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java index 17740c9f40d..fbeafdc3f6f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java @@ -13,7 +13,6 @@ import java.time.Clock; * A trace utility that can serialize/deserialize to/from {@link Slime} * * @author Ulf Lilleengen - * @since 5.3 */ public class Trace { private static final String TRACE_TRACELOG = "traceLog"; diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index d7653572773..8f856ff4771 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -33,6 +33,7 @@ import java.util.regex.Pattern; * Utilities for mangling config text, finding md5sums, finding name and namespace in .def files etc. */ public class ConfigUtils { + /* Patterns used for finding ranges in config definitions */ private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); @@ -88,6 +89,7 @@ public class ConfigUtils { /** * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * * @param str String to strip spaces from * @return String with spaces stripped */ @@ -103,15 +105,15 @@ public class ConfigUtils { } if (!inSpaceSequence) { // start of space sequence - inSpaceSequence=true; + inSpaceSequence = true; ret.append(" "); } } else { if (inSpaceSequence) { - inSpaceSequence=false; + inSpaceSequence = false; } - if (c=='\"') { - inQuotes=!inQuotes; + if (c == '\"') { + inQuotes = !inQuotes; } ret.append(c); } @@ -121,7 +123,7 @@ public class ConfigUtils { /** * Computes Md5 hash of a list of strings with the contents of a def-file. - * + * <p> * Each string is normalized according to the * rules of Vespa config definition files before they are used: * <ol> @@ -132,12 +134,12 @@ public class ConfigUtils { * <li>Remove 'version=<version-number>'</li> * </ol> * - * @param lines A list of lines constituting a def-file + * @param lines A list of lines constituting a def-file * @return the Md5 hash of the list, with lowercase letters */ public static String getDefMd5(List<String> lines) { List<String> linesCopy = new ArrayList<>(lines); - for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) { + for (Iterator<String> it = linesCopy.iterator(); it.hasNext(); ) { String line = it.next().trim(); if (! line.startsWith("#") && ! line.equals("")) { if (line.startsWith("version")) { @@ -169,7 +171,7 @@ public class ConfigUtils { } if (line.length() > 0) { line = stripSpaces(line); - m = spaceBeforeCommaPatter.matcher(line); + m = spaceBeforeCommaPatter.matcher(line); line = m.replaceAll(","); // Remove space before comma (for enums) sb.append(line).append("\n"); } @@ -188,7 +190,7 @@ public class ConfigUtils { public static String getDefNamespace(Reader in) { List<String> defLines = getDefLines(in); String defPackage = getDefKeyword(defLines, "package"); - if (! defPackage.isEmpty()) return defPackage; + if (!defPackage.isEmpty()) return defPackage; return getDefKeyword(defLines, "namespace"); } @@ -285,7 +287,7 @@ public class ConfigUtils { */ public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException { String[] fileName = file.getName().split("\\."); - assert(fileName.length >= 2); + assert (fileName.length >= 2); String name = fileName[fileName.length - 2]; byte[] content = IOUtils.readFileBytes(file); @@ -295,7 +297,7 @@ public class ConfigUtils { /** * Creates a ConfigDefinitionKey from a name and the content of a config definition * - * @param name the name of the config definition + * @param name the name of the config definition * @param content content of a config definition * @return a ConfigDefinitionKey */ @@ -306,6 +308,7 @@ public class ConfigUtils { /** * Escapes a config value according to the cfg format. + * * @param input the string to escape * @return the escaped string */ @@ -350,10 +353,10 @@ public class ConfigUtils { * Loop through values and return the first one that is set and non-empty. * * @param defaultValue The default value to use if no environment variables are set. - * @param envVars one or more environment variable strings + * @param envVars one or more environment variable strings * @return a String with the value of the environment variable */ - public static String getEnvValue(String defaultValue, String ... envVars) { + public static String getEnvValue(String defaultValue, String... envVars) { String value = null; for (String envVar : envVars) { if (value == null || value.isEmpty()) { diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java index d4280c6300f..c8693227920 100644 --- a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; /** * @author hmusum * @author Vegard Sjonfjell - * @since 5.1 */ public class CfgConfigPayloadBuilderTest { @@ -135,7 +134,7 @@ public class CfgConfigPayloadBuilderTest { " 'boolVal': 'true'", " },", " {", - " 'stringVal': 'blue a=\\\'escaped\\\'',", + " 'stringVal': 'blue a=\\'escaped\\'',", " 'boolVal': 'false'", " }", " ],", @@ -182,7 +181,7 @@ public class CfgConfigPayloadBuilderTest { assertDeserializedConfigEqualsJson("a b=\"escaped\"", inputJson( "{", - " 'a': 'b=\\\'escaped\\\''", + " 'a': 'b=\\'escaped\\''", "}" ) ); @@ -305,7 +304,7 @@ public class CfgConfigPayloadBuilderTest { } private static void assertDeserializedConfigEqualsJson(String serializedConfig, String expectedJson) { - assertDeserializedConfigEqualsJson(Arrays.asList(serializedConfig), expectedJson); + assertDeserializedConfigEqualsJson(List.of(serializedConfig), expectedJson); } private static void assertDeserializedConfigEqualsJson(List<String> inputConfig, String expectedJson) { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java index 2d0a40bcdc4..49fd25b6476 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; /** * @author gjoranv - * @since 5.1.6 */ public class ConfigInstancePayloadTest { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java index 8d9b2d96b04..95ffb31fe7d 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java @@ -3,7 +3,6 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; import com.yahoo.foo.FunctionTestConfig; -import com.yahoo.config.codegen.DefLine; import com.yahoo.vespa.config.ConfigPayload; import org.junit.Test; @@ -18,14 +17,6 @@ import static org.junit.Assert.assertThat; */ public class ConfigInstanceSerializationTest { - private DefLine.Type stringType = new DefLine.Type("string"); - private DefLine.Type intType = new DefLine.Type("int"); - private DefLine.Type longType = new DefLine.Type("long"); - private DefLine.Type boolType = new DefLine.Type("bool"); - private DefLine.Type doubleType = new DefLine.Type("double"); - private DefLine.Type fileType = new DefLine.Type("file"); - private DefLine.Type refType = new DefLine.Type("reference"); - @Test public void require_symmetrical_serialization_and_deserialization_with_builder() { FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); @@ -40,218 +31,4 @@ public class ConfigInstanceSerializationTest { assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2))); } -/** Looks like everything in the commented block tests unused api's.. remove? - - @Test - public void testSerializeAgainstConfigDefinitionAllowNothing() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - List<String> payload = Validator.serialize(config, def); - Assert.assertEquals(payload.size(), 0); - } - - @Test - public void testSerializeAgainstConfigDefinitionAllLeaves() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("bool_val", LeafCNode.newInstance(boolType, def, "bool_val")); - def.children().put("bool_with_def", LeafCNode.newInstance(boolType, def, "bool_with_def")); - def.children().put("int_val", LeafCNode.newInstance(intType, def, "int_val")); - def.children().put("int_with_def", LeafCNode.newInstance(intType, def, "int_with_def")); - def.children().put("long_val", LeafCNode.newInstance(longType, def, "long_val")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("double_val", LeafCNode.newInstance(doubleType, def, "double_val")); - def.children().put("double_with_def", LeafCNode.newInstance(doubleType, def, "double_with_def")); - def.children().put("string_val", LeafCNode.newInstance(stringType, def, "string_val")); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("enum_val", LeafCNode.newInstance(enumType(new String[]{"FOO", "BAR", "FOOBAR"}), def, "enum_val")); - def.children().put("enumwithdef", LeafCNode.newInstance(enumType(new String[]{"FOO2", "BAR2", "FOOBAR2"}), def, "enumwithdef", "BAR2")); - def.children().put("refval", LeafCNode.newInstance(refType, def, "refval")); - def.children().put("refwithdef", LeafCNode.newInstance(refType, def, "refwithdef")); - def.children().put("fileVal", LeafCNode.newInstance(fileType, def, "fileVal")); - - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*bool_val false.*")); - Assert.assertTrue(plString.matches(".*bool_with_def true.*")); - Assert.assertTrue(plString.matches(".*int_val 5.*")); - Assert.assertTrue(plString.matches(".*int_with_def -14.*")); - Assert.assertTrue(plString.matches(".*long_val 12345678901.*")); - Assert.assertTrue(plString.matches(".*long_with_def -9876543210.*")); - Assert.assertTrue(plString.matches(".*double_val 41\\.23.*")); - Assert.assertTrue(plString.matches(".*double_with_def -12.*")); - Assert.assertTrue(plString.matches(".*string_val \"foo\".*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertTrue(plString.matches(".*enum_val FOOBAR.*")); - Assert.assertTrue(plString.matches(".*enumwithdef BAR2.*")); - Assert.assertTrue(plString.matches(".*refval \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*refwithdef \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*fileVal \"etc\".*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionSomeLeaves() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - // But not double_with_def etc, and no structs/arrays - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - } - - @Test - public void testSerializationAgainstConfigDefinitionAddedValsInDef() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("someotherstring", LeafCNode.newInstance(stringType, def, "someotherstring", "some other")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("some_other_long", LeafCNode.newInstance(longType, def, "some_other_long", "88")); - def.children().put("some_other_enum", LeafCNode.newInstance(enumType(new String[]{"hey", "ho", "lets", "go"}), def, "some_other_enum", "lets")); - def.children().put("int_val_nofdef", LeafCNode.newInstance(intType, def, "int_val_nodef", null)); - - // But not double_with_def etc, and no structs/arrays - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertTrue(plString.matches(".*.someotherstring \"some other\".*")); - Assert.assertTrue(plString.matches(".*some_other_long 88.*")); - Assert.assertTrue(plString.matches(".*some_other_enum lets.*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - Assert.assertFalse(plString.matches(".*int_val_nodef.*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionMismatchAllWays() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - - // Create all sorts of mismatches in the def schema used to serialize - InnerCNode def = new InnerCNode("function-test"); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("stringwithdef", LeafCNode.newInstance(intType, def, "stringwithdef")); - def.children().put("basicStruct", LeafCNode.newInstance(intType, def, "basicStruct")); - InnerCNode doubleValWrong = new InnerCNode("double_val"); - doubleValWrong.children().put("foo", LeafCNode.newInstance(intType, def, "foo")); - def.children().put("double_val", doubleValWrong); - InnerCNode myArray = new InnerCNode("myarray"); - myArray.children().put("intval", LeafCNode.newInstance(stringType, myArray, "foo")); - InnerCNode myStruct = new InnerCNode("myStruct"); - myStruct.children().put("a", LeafCNode.newInstance(stringType, myStruct, "foo")); - myArray.children().put("myStruct", myStruct); - def.children().put("myarray", myArray); - - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def.*")); - Assert.assertFalse(plString.matches(".*stringwithdef.*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - Assert.assertFalse(plString.matches(".*double_val.*")); - Assert.assertFalse(plString.matches(".*intval.*")); - Assert.assertFalse(plString.matches(".*\\.a.*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionComplex() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - - // Build a pretty complex def programatically - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("someUnknownStringNoDefault", LeafCNode.newInstance(stringType, def, "someUnknownStringNoDefault")); - InnerCNode basicStruct = new InnerCNode("basicStruct"); - basicStruct.children().put("foo", LeafCNode.newInstance(stringType, def, "foo")); // but not bar - InnerCNode rootStruct = new InnerCNode("rootStruct"); - InnerCNode inner1 = new InnerCNode("inner1"); - InnerCNode someUnknwonStruct = new InnerCNode("someUnknownStruct"); - InnerCNode someUnknownInner = new InnerCNode("someUnknownInner"); - InnerCNode innerArr = new InnerCNode("innerArr"); - rootStruct.children().put("inner1", inner1); - rootStruct.children().put("someUnknownStruct", someUnknwonStruct); - rootStruct.children().put("someUnknownInner", someUnknownInner); - rootStruct.children().put("innerArr", innerArr); - InnerCNode myarray = new InnerCNode("myarray"); - InnerCNode unknownInner = new InnerCNode("unknownInner"); - def.children().put("basicStruct", basicStruct); - def.children().put("rootStruct", rootStruct); - def.children().put("myarray", myarray); - def.children().put("unknownInner", unknownInner); - inner1.children().put("index", LeafCNode.newInstance(intType, inner1, "index")); - inner1.children().put("someUnknownInt", LeafCNode.newInstance(intType, inner1, "someUnknownInt", "-98")); - inner1.children().put("someUnknownIntNoDefault", LeafCNode.newInstance(intType, inner1, "someUnknownIntNoDefault")); - inner1.children().put("someUnknownEnum", LeafCNode.newInstance(enumType(new String[]{"goo", "go", "gorilla"}), inner1, "someUnknownEnum", "go")); - inner1.children().put("someUnknownEnumNoDefault", LeafCNode.newInstance(enumType(new String[]{"foo", "bar", "baz"}), inner1, "someUnknownEnumNoDefault")); - someUnknwonStruct.children().put("anint", LeafCNode.newInstance(intType, someUnknwonStruct, "anint", "3"));// But no instances of this in config - someUnknownInner.children().put("along", LeafCNode.newInstance(longType, someUnknownInner, "along", "234"));// No instance in config - innerArr.children().put("boolVal", LeafCNode.newInstance(boolType, innerArr, "boolVal")); - innerArr.children().put("someUnknownDouble", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDouble", "-675.789")); - innerArr.children().put("someUnknownDoubleNoDefault", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDoubleNoDefault")); - myarray.children().put("fileVal", LeafCNode.newInstance(fileType, myarray, "fileVal")); - myarray.children().put("stringval", new InnerCNode("stringval[]")); - // TODO make sure default for file is not allowed - //myarray.children().put("someUnknownFile", LeafCNode.newInstance(fileType, myarray, "someUnknownFile", "opt/")); - unknownInner.children().put("aDouble", LeafCNode.newInstance(doubleType, unknownInner, "aDouble", "1234")); - def.children().put("longarr", new InnerCNode("longarr[]")); - def.children().put("boolarr", new InnerCNode("boolarr[]")); - def.children().put("doublearr", new InnerCNode("doublearr[]")); - def.children().put("stringarr", new InnerCNode("stringarr[]")); - def.children().put("fileArr", new InnerCNode("fileArr[]")); - def.children().put("refarr", new InnerCNode("refarr[]")); - def.children().put("enumarr", new InnerCNode("enumarr[]")); - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertFalse(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertFalse(plString.matches(".*someUnknownStringNoDefault.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal etc.*")); - Assert.assertTrue(plString.matches(".*basicStruct\\.foo \"basicFoo\".*")); - Assert.assertFalse(plString.matches(".*basicStruct\\.bar.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.inner0.*")); - Assert.assertFalse(plString.matches(".*unknownInner.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownStruct.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownInner.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.inner1\\.name.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.index 12.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownInt -98.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownEnum go.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.boolVal true.*")); - Assert.assertFalse(plString.matches(".*someUnknownEnumNoDefault.*")); - Assert.assertFalse(plString.matches(".*someUnknownDoubleNoDefault.*")); - Assert.assertFalse(plString.matches(".*someUnknownIntNoDefault.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.someUnknownDouble -675.789.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.stringVal*")); - Assert.assertFalse(plString.matches(".*myarray\\[0\\].intval.*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].fileVal \"file0\".*")); - //assertTrue(plString.matches(".*myarray\\[0\\].someUnknownFile \"opt/\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[0\\] \"baah\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[1\\] \"yikes\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[1\\].fileVal \"file1\".*")); - Assert.assertFalse(plString.matches(".*myarray\\[1\\].enumVal.*")); - Assert.assertFalse(plString.matches(".*myarray\\[1\\].refVal.*")); - Assert.assertTrue(plString.matches(".*boolarr\\[0\\] false.*")); - Assert.assertTrue(plString.matches(".*longarr\\[0\\] 9223372036854775807.*")); - Assert.assertTrue(plString.matches(".*longarr\\[1\\] -9223372036854775808.*")); - Assert.assertTrue(plString.matches(".*doublearr\\[0\\] 2344\\.0.*")); - Assert.assertTrue(plString.matches(".*doublearr\\[1\\] 123\\.0.*")); - Assert.assertTrue(plString.matches(".*stringarr\\[0\\] \"bar\".*")); - Assert.assertTrue(plString.matches(".*enumarr\\[0\\] VALUES.*")); - Assert.assertTrue(plString.matches(".*refarr\\[0\\] \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*refarr\\[1\\] \\:parent.*")); - Assert.assertTrue(plString.matches(".*refarr\\[2\\] parent\\:.*")); - Assert.assertTrue(plString.matches(".*fileArr\\[0\\] \"bin\".*")); - } - - private DefLine.Type enumType(String[] strings) { - return new DefLine.Type("enum").setEnumArray(strings); - } -**/ } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java index ee8682efe3c..7c08d0175a1 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java @@ -21,9 +21,9 @@ import static org.junit.Assert.fail; /** * @author Ulf Lilleengen * @author Vegard Sjonfjell - * @since 5.1 */ public class ConfigInstanceSerializerTest { + @Test public void test_that_leaf_types_are_serialized_to_json_types() { SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); @@ -225,4 +225,5 @@ public class ConfigInstanceSerializerTest { assertJsonEquals(baos.toString(), expectedJson); } + } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java index 1da53e4c3b9..4da0c3f51e0 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java @@ -12,7 +12,7 @@ import static org.junit.Assert.fail; /** * Tests different aspects of the ConfigInstance class and its underlying Nodes. * - * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + * @author gjoranv */ public class ConfigInstanceTest { private ConfigSourceSet sourceSet = new ConfigSourceSet("config-instance-test"); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java index 078be819d33..3bdaee09eaf 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java @@ -22,7 +22,6 @@ import static com.yahoo.foo.FunctionTestConfig.*; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigInstanceUtilTest { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java index 35f4f14b4f4..a80da63afc9 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java @@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigInterruptedExceptionTest { + @Test public void require_that_throwable_is_preserved() { ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo")); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java index a120f19c17b..48a51bbc02c 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java @@ -10,9 +10,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigSetTest { + @Test public void testToString() { ConfigSet set = new ConfigSet(); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java index 38d4a6a4571..0879c330f45 100755 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.*; * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> */ public class ConfigSourceSetTest { + @Test public void testEquals() { assertEquals(new ConfigSourceSet(), new ConfigSourceSet()); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java index 43e3cf658f3..555434837c8 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java @@ -12,9 +12,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigURITest { + @Test public void testDefaultUri() { ConfigURI uri = ConfigURI.createFromId("foo"); diff --git a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java index 85346264837..1bcf09a4028 100644 --- a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; * @author hmusum */ public class DefaultConfigTest { + static final String CONFIG_ID = "raw:" + "nondefaultstring ####-------missing--------\n" + "defaultstring \"thedefault\"\n" + diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index cb25de89bfb..e9dc9cf7b98 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; @@ -14,14 +17,16 @@ import com.yahoo.vespa.config.protocol.CompressionType; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * * Test cases for the "generic" (class-less) subscription mechanism. * * @author Ulf Lilleengen - * @since 5.1 */ public class GenericConfigSubscriberTest { @@ -29,9 +34,9 @@ public class GenericConfigSubscriberTest { public void testSubscribeGeneric() { Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); ConfigSourceSet sourceSet = new ConfigSourceSet("blabla"); - requesters.put(sourceSet, JRTConfigRequester.get(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); + requesters.put(sourceSet, new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); - final List<String> defContent = Arrays.asList("myVal int"); + final List<String> defContent = List.of("myVal int"); GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), defContent, sourceSet, JRTConfigRequesterTest.getTestTimingValues()); assertTrue(sub.nextConfig()); assertTrue(handle.isChanged()); @@ -44,32 +49,35 @@ public class GenericConfigSubscriberTest { public void testGenericRequesterPooling() { ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); - JRTConfigRequester req1 = JRTConfigRequester.get(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); - JRTConfigRequester req2 = JRTConfigRequester.get(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req1 = new JRTConfigRequester(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req2 = new JRTConfigRequester(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); requesters.put(source1, req1); requesters.put(source2, req2); GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78"); assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79"); - } @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid1() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null); + createSubscriber().subscribe(null, null); } @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid2() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null, 0L); + createSubscriber().subscribe(null, null, 0L); } @Test(expected=UnsupportedOperationException.class) public void testOverriddenSubscribeInvalid3() { - GenericConfigSubscriber sub = new GenericConfigSubscriber(); - sub.subscribe(null, null, ""); + createSubscriber().subscribe(null, null, ""); + } + + private GenericConfigSubscriber createSubscriber() { + return new GenericConfigSubscriber(Map.of( + new ConfigSourceSet("blabla"), + new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues()))); } + } diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java index 5e60c273ee4..430de894629 100644 --- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -11,6 +11,7 @@ import static org.junit.Assert.assertThat; * @author gjoranv */ public class NamespaceTest { + @Test public void verifyConfigClassWithExplicitNamespace() { NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java index 6512b5ea29e..f1bd164c874 100644 --- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertEquals; * @author Harald Musum */ public class UnicodeTest { + /** * Reads a config from a file which is exactly like one returned from * the config server given only default values for this config. diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java index 2f550fc8e1e..71e7a9f08d7 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1.7 */ public class FileConfigSubscriptionTest { private File TEST_TYPES_FILE; diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java index c77985c91d8..137d2894164 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertThat; * SEO keywords: test override() on builders. overrideTest, testOverride * * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigBuilderMergeTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java index bb65fdaa153..fce61cc802c 100755 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.*; * @author hmusum */ public class ConfigCacheKeyTest { + @Test public void testConfigCacheKey() { final String defMd5 = "md5"; diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java index dba73223097..810b9f58829 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java @@ -12,7 +12,6 @@ import java.io.IOException; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; - /** * Unit tests for ConfigDefinitionBuilder. * diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java index 3cc030d944b..5198759d3e2 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java @@ -24,7 +24,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigFileFormatterTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java index eb0c47e3b0a..c13d3ec9b9b 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigPayloadBuilderTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java index f1b0adc03e7..b3db6e2ab43 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java @@ -24,8 +24,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author Ulf Lilleengen 3 - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigPayloadTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java index e67489b4030..6ab3059f1c0 100644 --- a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java @@ -16,9 +16,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class DefaultValueApplierTest { + public Slime apply(Slime slime, String ... extraFields) { StringBuilder defBuilder = new StringBuilder(); defBuilder.append("namespace=test").append("\n"); diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java index 8e6cc1fc6a0..64210eaa4b8 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java @@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat; /** * @author hmusum - * @since 5.1.9 */ public class ErrorCodeTest { + @Test public void basic() { assertThat(ErrorCode.getName(ErrorCode.INTERNAL_ERROR), is("INTERNAL_ERROR")); diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java index 450d7e8d6d8..a0b2c1185a7 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java @@ -8,7 +8,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ErrorTypeTest { @@ -32,4 +31,5 @@ public class ErrorTypeTest { assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL)); assertThat(ErrorType.getErrorType(0xdeadc0de), is(ErrorType.FATAL)); } + } diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java index ee7845086c2..2c60ff95fee 100644 --- a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java @@ -13,9 +13,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class GenericConfigBuilderTest { + @Test public void require_that_builder_can_be_overridden() throws IOException { ConfigPayloadBuilder ba = new ConfigPayloadBuilder(); diff --git a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java index 8a403b45003..cc46301e869 100644 --- a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java @@ -70,7 +70,7 @@ public class JRTConnectionPoolTest { // Tests that the number of times each connection is used is close to equal private void assertConnectionDistributionIsFair(Map<String, Integer> connectionsUsedPerHost) { - double devianceDueToRandomSourceSelection = 0.13; + double devianceDueToRandomSourceSelection = 0.14; final int size = 1000; int minHostCount = (int) (size/2 * (1 - devianceDueToRandomSourceSelection)); int maxHostCount = (int) (size/2 * (1 + devianceDueToRandomSourceSelection)); diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java index a3125c73bea..805efc90ade 100644 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java @@ -15,7 +15,6 @@ import java.nio.file.Files; * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge. * * @author Ulf Lilleengen - * @since 5.12 */ public class LZ4CompressionTest { private static LZ4Factory factory = LZ4Factory.safeInstance(); diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java index 06bbaad4271..635d6a96a7c 100644 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java @@ -9,9 +9,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.19 */ public class LZ4PayloadCompressorTest { + @Test public void testCompression() { assertCompression("hei hallo der"); diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index a564fea8b2e..3f6c54ea46e 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -113,11 +113,11 @@ public class RawConfigTest { assertThat(config.getDefMd5(), is(defMd5)); config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty()); assertNull(config.getDefMd5()); - config = new RawConfig(key, null, payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, null, payload, configMd5, generation, false,List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty()); assertThat(config.getDefMd5(), is("")); - config = new RawConfig(key, "", payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); } diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java deleted file mode 100644 index 3b97c360c4b..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; - -/** - * Note: Most of the functionality is tested implicitly by other tests - * - * @author hmusum - */ -public class TimingValuesTest { - @Test - public void basic() { - TimingValues tv = new TimingValues(); - TimingValues tv2 = new TimingValues(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1); - assertThat(tv.getRandom(), is(not(tv2.getRandom()))); - TimingValues copy = new TimingValues(tv2); - assertThat(copy.toString(), is(tv2.toString())); // No equals method, just using toString to compare - } -} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java index 91adc544d88..a56c7ef2daa 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.config.protocol; import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.text.StringUtilities; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.LZ4PayloadCompressor; @@ -12,10 +9,10 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.StringReader; -import java.util.List; +import java.nio.charset.StandardCharsets; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -27,17 +24,16 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_is_initialized() throws IOException { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); - ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, false, "mymd5"); - List<String> payload = response.getLegacyPayload(); + ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, 3, false, "mymd5"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + String payload = baos.toString(StandardCharsets.UTF_8); assertNotNull(payload); - assertThat(payload.size(), is(6)); - assertThat(payload.get(0), is("boolval false")); + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload); assertThat(response.getGeneration(), is(3L)); assertThat(response.getConfigMd5(), is("mymd5")); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -45,19 +41,15 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_decompresses_on_serialize() throws IOException { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); Utf8Array data = configPayload.toUtf8Array(true); Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.getBytes())); - ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); - List<String> payload = response.getLegacyPayload(); + ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + String payload = baos.toString(StandardCharsets.UTF_8); assertNotNull(payload); - assertThat(payload.size(), is(6)); - assertThat(payload.get(0), is("boolval false")); - assertThat(response.getGeneration(), is(3L)); - assertThat(response.getConfigMd5(), is("mymd5")); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java deleted file mode 100644 index 75ba1392fd1..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.protocol; - -import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.config.subscription.ConfigSet; -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.config.subscription.impl.GenericConfigSubscriber; -import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.config.subscription.impl.JRTConfigSubscription; -import com.yahoo.config.subscription.impl.MockConnection; -import com.yahoo.jrt.Request; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.JsonDecoder; -import com.yahoo.slime.Slime; -import com.yahoo.test.ManualClock; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.ErrorCode; -import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; -import com.yahoo.vespa.config.util.ConfigUtils; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - * @since 5.3 - */ -public abstract class JRTConfigRequestBase { - - protected String defName = "mydef"; - protected String defNamespace = "my.name.space"; - protected String hostname = "myhost"; - protected String configId = "config/id"; - protected String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1"; - protected long currentGeneration = 3; - protected final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24")); - protected long timeout = 5000; - protected Trace trace ; - protected String configMd5 = ConfigUtils.getMd5(createPayload().getData()); - protected JRTClientConfigRequest clientReq; - protected JRTServerConfigRequest serverReq; - - @Before - public void setupRequest() throws IOException { - clientReq = createReq(); - serverReq = createReq(clientReq.getRequest()); - assertTrue(serverReq.validateParameters()); - } - - private JRTClientConfigRequest createReq() throws IOException { - trace = Trace.createNew(3, new ManualClock()); - trace.trace(1, "hei"); - return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace); - } - - private JRTClientConfigRequest createReq(Payload payload) throws IOException { - trace = Trace.createNew(3, new ManualClock()); - trace.trace(1, "hei"); - return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); - } - - protected abstract JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, - String hostname, String configId, String configMd5, - long currentGeneration, long timeout, Trace trace); - protected abstract JRTServerConfigRequest createReq(Request request); - protected abstract JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew); - protected abstract JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew); - - protected void request_is_parsed_base() { - String [] expectedContent = new String[]{ - "namespace=my.name.space", - "myfield string" - }; - System.out.println(serverReq.toString()); - assertThat(serverReq.getConfigKey().getName(), is(defName)); - assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace)); - assertThat(serverReq.getConfigKey().getMd5(), is(defMd5)); - assertThat(serverReq.getConfigKey().getConfigId(), is(configId)); - assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent)); - assertFalse(serverReq.noCache()); - assertTrue(serverReq.getRequestTrace().toString().contains("hi")); - assertThat(serverReq.getRequestConfigMd5(), is(configMd5)); - assertThat(serverReq.getRequestGeneration(), is(currentGeneration)); - } - - @Test - public void delay_mechanisms_functions() { - assertFalse(serverReq.isDelayedResponse()); - serverReq.setDelayedResponse(true); - assertTrue(serverReq.isDelayedResponse()); - serverReq.setDelayedResponse(false); - assertFalse(serverReq.isDelayedResponse()); - } - - public JRTServerConfigRequest next_request_is_correct_base() { - String [] expectedContent = new String[]{ - "namespace=my.name.space", - "myfield string" - }; - JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest()); - assertThat(next.getConfigKey().getName(), is(defName)); - assertThat(next.getConfigKey().getNamespace(), is(defNamespace)); - assertThat(next.getConfigKey().getMd5(), is(defMd5)); - assertThat(next.getConfigKey().getConfigId(), is(configId)); - assertThat(next.getDefContent().asStringArray(), is(expectedContent)); - assertFalse(next.noCache()); - assertThat(next.getTimeout(), is(6L)); - assertThat(next.getTimeout(), is(6L)); - return next; - } - - - @Test - public void next_request_when_error_is_correct() { - serverReq.addOkResponse(createPayload(), 999999, false, "newmd5"); - serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message"); - System.out.println(serverReq); - JRTClientConfigRequest next = clientReq.nextRequest(6); - System.out.println(next); - // Should use config md5 and generation from the request, not the response - // when there are errors - assertThat(next.getRequestConfigMd5(), is(clientReq.getRequestConfigMd5())); - assertThat(next.getRequestGeneration(), is(clientReq.getRequestGeneration())); - } - - @Test - public void ok_response_is_added() { - Payload payload = createPayload("vale"); - String md5 = ConfigUtils.getMd5(payload.getData()); - long generation = 4L; - serverReq.addOkResponse(payload, generation, false, md5); - assertTrue(clientReq.validateResponse()); - assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString())); - assertThat(clientReq.getNewGeneration(), is(4L)); - assertThat(clientReq.getNewConfigMd5(), is(md5)); - assertTrue(clientReq.hasUpdatedConfig()); - assertTrue(clientReq.hasUpdatedGeneration()); - } - - @Test - public void error_response_adds_common_elements() { - serverReq.addErrorResponse(ErrorCode.APPLICATION_NOT_LOADED, ErrorCode.getName(ErrorCode.APPLICATION_NOT_LOADED)); - assertThat(serverReq.getRequest().returnValues().size(), is(1)); - Slime data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(serverReq.getRequest().returnValues().get(0).asString())); - Inspector response = data.get(); - assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAME).asString(), is(defName)); - assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAMESPACE).asString(), is(defNamespace)); - assertThat(response.field(SlimeResponseData.RESPONSE_DEF_MD5).asString(), is(defMd5)); - assertThat(response.field(SlimeResponseData.RESPONSE_CONFIGID).asString(), is(configId)); - assertThat(response.field(SlimeResponseData.RESPONSE_CLIENT_HOSTNAME).asString(), is(hostname)); - Trace t = Trace.fromSlime(response.field(SlimeResponseData.RESPONSE_TRACE)); - assertThat(t.toString(), is(trace.toString())); - } - - @Test - public void generation_only_is_updated() { - Payload payload = createPayload(); - serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData())); - boolean value = clientReq.validateResponse(); - assertTrue(clientReq.errorMessage(), value); - assertFalse(clientReq.hasUpdatedConfig()); - assertTrue(clientReq.hasUpdatedGeneration()); - } - - protected static Payload createPayload() { - return createPayload("bar"); - } - - private static Payload createPayload(String value) { - Slime slime = new Slime(); - slime.setObject().setString("myfield", value); - return Payload.from(new ConfigPayload(slime)); - } - - @Test - public void nothing_is_updated() { - Payload payload = createPayload(); - serverReq.addOkResponse(payload, currentGeneration, false, configMd5); - assertTrue(clientReq.validateResponse()); - assertFalse(clientReq.hasUpdatedConfig()); - assertFalse(clientReq.hasUpdatedGeneration()); - } - - @Test - public void payload_is_empty() throws IOException { - Payload payload = Payload.from(ConfigPayload.empty()); - clientReq = createReq(payload); - serverReq = createReq(clientReq.getRequest()); - serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData())); - boolean val = clientReq.validateResponse(); - assertTrue(clientReq.errorMessage(), val); - assertFalse(clientReq.hasUpdatedConfig()); - assertFalse(clientReq.hasUpdatedGeneration()); - } - - @Test - public void request_interface_is_implemented() { - JRTClientConfigRequest request = clientReq; - assertFalse(request.containsPayload()); - assertFalse(request.isError()); - assertThat(request.errorCode(), is(clientReq.getRequest().errorCode())); - assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage())); - assertNotNull(request.getRequest()); - assertFalse(request.validateResponse()); - //assertNull(request.getNewPayload().getData()); - assertThat(request.getTimeout(), is(timeout)); - assertFalse(request.hasUpdatedConfig()); - assertFalse(request.hasUpdatedGeneration()); - } - - @Test - public void created_from_subscription() { - ConfigSubscriber subscriber = new ConfigSubscriber(); - JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, new ConfigSet(), new TimingValues()); - JRTClientConfigRequest request = createReq(sub, Trace.createNew(9)); - assertThat(request.getConfigKey().getName(), is(SimpletypesConfig.CONFIG_DEF_NAME)); - JRTServerConfigRequest serverRequest = createReq(request.getRequest()); - assertTrue(serverRequest.validateParameters()); - } - - @Test - public void created_from_existing_subscription() { - System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", getProtocolVersion()); - MockConnection connection = new MockConnection(new MockConnection.AbstractResponseHandler() { - @Override - protected void createResponse() { - JRTServerConfigRequest serverRequest = createReq(request); - serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5); - } - }); - - ConfigSourceSet src = new ConfigSourceSet(); - ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, JRTConfigRequester.get(connection, new TimingValues()))); - JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues()); - sub.subscribe(120_0000); - assertTrue(sub.nextConfig(120_0000)); - sub.close(); - JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew()); - SimpletypesConfig config = sub.getConfigState().getConfig(); - assertThat(nextReq.getRequestConfigMd5(), is(config.getConfigMd5())); - assertThat(nextReq.getRequestGeneration(), is(currentGeneration)); - System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", ""); - } - - protected abstract String getProtocolVersion(); - - @Test - public void created_from_raw() throws IOException { - RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5); - long serverTimeout = 100000L; - JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9)); - assertThat(request.getConfigKey().getName(), is(defName)); - JRTServerConfigRequest serverRequest = createReq(request.getRequest()); - assertTrue(serverRequest.validateParameters()); - assertThat(serverRequest.getTimeout(), is(serverTimeout)); - assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent())); - } - - - @Test - public void parameters_are_validated() throws IOException { - assertTrue(serverReq.validateParameters()); - assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace)); - assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace)); - } - - private void assertValidationFail(JRTClientConfigRequest req) { - assertFalse(createReq(req.getRequest()).validateParameters()); - } -} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index d50a92efc1a..4d1ba2f8793 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -1,59 +1,58 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.config.subscription.impl.MockConnection; import com.yahoo.foo.SimpletypesConfig; import com.yahoo.config.subscription.impl.JRTConfigSubscription; import com.yahoo.jrt.Request; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.test.ManualClock; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.util.ConfigUtils; +import org.junit.Before; import org.junit.Test; import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.19 */ -public class JRTConfigRequestV3Test extends JRTConfigRequestBase { +public class JRTConfigRequestV3Test { - @Override - protected JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, - String hostname, String configId, String configMd5, - long currentGeneration, long timeout, Trace trace) { - return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5), - DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")), - hostname, - configMd5, - currentGeneration, - timeout, - trace, - CompressionType.LZ4, - vespaVersion); - } - - @Override - protected JRTServerConfigRequest createReq(Request request) { - return JRTServerConfigRequestV3.createFromRequest(request); - } - - @Override - protected JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew) { - return JRTClientConfigRequestV3.createFromSub(sub, aNew, CompressionType.LZ4, vespaVersion); - } + private final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24")); + private String defName = "mydef"; + private String defNamespace = "my.name.space"; + private String hostname = "myhost"; + private String configId = "config/id"; + private String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1"; + private long currentGeneration = 3; + private long timeout = 5000; + private Trace trace ; + private String configMd5 = ConfigUtils.getMd5(createPayload().getData()); + private JRTClientConfigRequest clientReq; + private JRTServerConfigRequest serverReq; - @Override - protected JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew) { - return JRTClientConfigRequestV3.createFromRaw(rawConfig, serverTimeout, aNew, CompressionType.LZ4, vespaVersion); - } - - @Override - protected String getProtocolVersion() { - return "3"; + @Before + public void setupRequest() { + clientReq = createReq(); + serverReq = createReq(clientReq.getRequest()); + assertTrue(serverReq.validateParameters()); } @Test @@ -71,11 +70,250 @@ public class JRTConfigRequestV3Test extends JRTConfigRequestBase { @Test public void emptypayload() { ConfigPayload payload = ConfigPayload.empty(); - SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, false, ConfigUtils.getMd5(payload)); + SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, 0, false, ConfigUtils.getMd5(payload)); serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5()); assertTrue(clientReq.validateResponse()); assertTrue(clientReq.hasUpdatedGeneration()); assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is("{}")); assertFalse(clientReq.responseIsInternalRedeploy()); } + + @Test + public void delay_mechanisms_function() { + assertFalse(serverReq.isDelayedResponse()); + serverReq.setDelayedResponse(true); + assertTrue(serverReq.isDelayedResponse()); + serverReq.setDelayedResponse(false); + assertFalse(serverReq.isDelayedResponse()); + } + + @Test + public void next_request_when_error_is_correct() { + serverReq.addOkResponse(createPayload(), 999999, false, "newmd5"); + serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message"); + System.out.println(serverReq); + JRTClientConfigRequest next = clientReq.nextRequest(6); + System.out.println(next); + // Should use config md5 and generation from the request, not the response + // when there are errors + assertThat(next.getRequestConfigMd5(), is(clientReq.getRequestConfigMd5())); + assertThat(next.getRequestGeneration(), is(clientReq.getRequestGeneration())); + } + + @Test + public void ok_response_is_added() { + Payload payload = createPayload("vale"); + String md5 = ConfigUtils.getMd5(payload.getData()); + long generation = 4L; + serverReq.addOkResponse(payload, generation, false, md5); + assertTrue(clientReq.validateResponse()); + assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString())); + assertThat(clientReq.getNewGeneration(), is(4L)); + assertThat(clientReq.getNewConfigMd5(), is(md5)); + assertTrue(clientReq.hasUpdatedConfig()); + assertTrue(clientReq.hasUpdatedGeneration()); + } + + @Test + public void error_response_adds_common_elements() { + serverReq.addErrorResponse(ErrorCode.APPLICATION_NOT_LOADED, ErrorCode.getName(ErrorCode.APPLICATION_NOT_LOADED)); + assertThat(serverReq.getRequest().returnValues().size(), is(1)); + Slime data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(serverReq.getRequest().returnValues().get(0).asString())); + Inspector response = data.get(); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAME).asString(), is(defName)); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAMESPACE).asString(), is(defNamespace)); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_MD5).asString(), is(defMd5)); + assertThat(response.field(SlimeResponseData.RESPONSE_CONFIGID).asString(), is(configId)); + assertThat(response.field(SlimeResponseData.RESPONSE_CLIENT_HOSTNAME).asString(), is(hostname)); + Trace t = Trace.fromSlime(response.field(SlimeResponseData.RESPONSE_TRACE)); + assertThat(t.toString(), is(trace.toString())); + } + + @Test + public void generation_only_is_updated() { + Payload payload = createPayload(); + serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData())); + boolean value = clientReq.validateResponse(); + assertTrue(clientReq.errorMessage(), value); + assertFalse(clientReq.hasUpdatedConfig()); + assertTrue(clientReq.hasUpdatedGeneration()); + } + + @Test + public void nothing_is_updated() { + Payload payload = createPayload(); + serverReq.addOkResponse(payload, currentGeneration, false, configMd5); + assertTrue(clientReq.validateResponse()); + assertFalse(clientReq.hasUpdatedConfig()); + assertFalse(clientReq.hasUpdatedGeneration()); + } + + @Test + public void payload_is_empty() { + Payload payload = Payload.from(ConfigPayload.empty()); + clientReq = createReq(payload); + serverReq = createReq(clientReq.getRequest()); + serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData())); + boolean val = clientReq.validateResponse(); + assertTrue(clientReq.errorMessage(), val); + assertFalse(clientReq.hasUpdatedConfig()); + assertFalse(clientReq.hasUpdatedGeneration()); + } + + @Test + public void request_interface_is_implemented() { + JRTClientConfigRequest request = clientReq; + assertFalse(request.isError()); + assertThat(request.errorCode(), is(clientReq.getRequest().errorCode())); + assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage())); + assertNotNull(request.getRequest()); + assertFalse(request.validateResponse()); + //assertNull(request.getNewPayload().getData()); + assertThat(request.getTimeout(), is(timeout)); + assertFalse(request.hasUpdatedConfig()); + assertFalse(request.hasUpdatedGeneration()); + } + + @Test + public void created_from_subscription() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, new ConfigSet(), new TimingValues()); + JRTClientConfigRequest request = createReq(sub, Trace.createNew(9)); + assertThat(request.getConfigKey().getName(), is(SimpletypesConfig.CONFIG_DEF_NAME)); + JRTServerConfigRequest serverRequest = createReq(request.getRequest()); + assertTrue(serverRequest.validateParameters()); + } + + @Test + public void created_from_existing_subscription() { + MockConnection connection = new MockConnection(new MockConnection.AbstractResponseHandler() { + @Override + public void createResponse() { + JRTServerConfigRequest serverRequest = createReq(request); + serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5); + } + }); + + ConfigSourceSet src = new ConfigSourceSet(); + ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, new JRTConfigRequester(connection, new TimingValues()))); + JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues()); + sub.subscribe(120_0000); + assertTrue(sub.nextConfig(120_0000)); + sub.close(); + JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew()); + SimpletypesConfig config = sub.getConfigState().getConfig(); + assertThat(nextReq.getRequestConfigMd5(), is(config.getConfigMd5())); + assertThat(nextReq.getRequestGeneration(), is(currentGeneration)); + } + + @Test + public void created_from_raw() { + RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5); + long serverTimeout = 100000L; + JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9)); + assertThat(request.getConfigKey().getName(), is(defName)); + JRTServerConfigRequest serverRequest = createReq(request.getRequest()); + assertTrue(serverRequest.validateParameters()); + assertThat(serverRequest.getTimeout(), is(serverTimeout)); + assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent())); + } + + @Test + public void parameters_are_validated() { + assertTrue(serverReq.validateParameters()); + assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace)); + } + + private void assertValidationFail(JRTClientConfigRequest req) { + assertFalse(createReq(req.getRequest()).validateParameters()); + } + + private static Payload createPayload() { + return createPayload("bar"); + } + + private static Payload createPayload(String value) { + Slime slime = new Slime(); + slime.setObject().setString("myfield", value); + return Payload.from(new ConfigPayload(slime)); + } + + private JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, + String hostname, String configId, String configMd5, + long currentGeneration, long timeout, Trace trace) { + return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5), + DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")), + hostname, + configMd5, + currentGeneration, + timeout, + trace, + CompressionType.LZ4, + vespaVersion); + } + + private JRTServerConfigRequest createReq(Request request) { + return JRTServerConfigRequestV3.createFromRequest(request); + } + + private JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew) { + return JRTClientConfigRequestV3.createFromSub(sub, aNew, CompressionType.LZ4, vespaVersion); + } + + private JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew) { + return JRTClientConfigRequestV3.createFromRaw(rawConfig, serverTimeout, aNew, CompressionType.LZ4, vespaVersion); + } + + private JRTClientConfigRequest createReq() { + trace = Trace.createNew(3, new ManualClock()); + trace.trace(1, "hei"); + return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace); + } + + private JRTClientConfigRequest createReq(Payload payload) { + trace = Trace.createNew(3, new ManualClock()); + trace.trace(1, "hei"); + return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); + } + + private void request_is_parsed_base() { + String [] expectedContent = new String[]{ + "namespace=my.name.space", + "myfield string" + }; + System.out.println(serverReq.toString()); + assertThat(serverReq.getConfigKey().getName(), is(defName)); + assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace)); + assertThat(serverReq.getConfigKey().getMd5(), is(defMd5)); + assertThat(serverReq.getConfigKey().getConfigId(), is(configId)); + assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent)); + assertFalse(serverReq.noCache()); + assertTrue(serverReq.getRequestTrace().toString().contains("hi")); + assertThat(serverReq.getRequestConfigMd5(), is(configMd5)); + assertThat(serverReq.getRequestGeneration(), is(currentGeneration)); + } + + private JRTServerConfigRequest next_request_is_correct_base() { + String [] expectedContent = new String[]{ + "namespace=my.name.space", + "myfield string" + }; + JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest()); + assertThat(next.getConfigKey().getName(), is(defName)); + assertThat(next.getConfigKey().getNamespace(), is(defNamespace)); + assertThat(next.getConfigKey().getMd5(), is(defMd5)); + assertThat(next.getConfigKey().getConfigId(), is(configId)); + assertThat(next.getDefContent().asStringArray(), is(expectedContent)); + assertFalse(next.noCache()); + assertThat(next.getTimeout(), is(6L)); + assertThat(next.getTimeout(), is(6L)); + return next; + } + } diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java index e5fc5190ad1..c52245ecb35 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java @@ -15,7 +15,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.21 */ public class PayloadTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java index f2d9edc7c35..22baab85c1c 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java @@ -21,9 +21,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceSerializerTest { + @Test public void test_serializer() throws IOException { TraceNode root = new TraceNode(null, 1); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java index 596983eebb8..4b191d85121 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java @@ -9,7 +9,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.3 */ public class TraceTest { @@ -58,4 +57,5 @@ public class TraceTest { assertTrue(trace2Str.contains("barbaz")); assertTrue(trace2Str.contains("quux")); } + } diff --git a/config/src/vespa/config/configgen/value_converter.cpp b/config/src/vespa/config/configgen/value_converter.cpp index 1f78e78cc32..06086939d58 100644 --- a/config/src/vespa/config/configgen/value_converter.cpp +++ b/config/src/vespa/config/configgen/value_converter.cpp @@ -7,9 +7,7 @@ using namespace vespalib; using namespace vespalib::slime; -namespace config { - -namespace internal { +namespace config::internal { template<> int32_t convertValue(const ::vespalib::slime::Inspector & __inspector) { @@ -64,4 +62,3 @@ requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Insp } -} diff --git a/config/src/vespa/config/configgen/value_converter.h b/config/src/vespa/config/configgen/value_converter.h index d2d04169c1d..6ad67bf6f86 100644 --- a/config/src/vespa/config/configgen/value_converter.h +++ b/config/src/vespa/config/configgen/value_converter.h @@ -5,9 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/data/slime/inspector.h> -namespace config { - -namespace internal { +namespace config::internal { void requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector); @@ -43,7 +41,4 @@ struct ValueConverter { } }; -} // namespace internal - -} // namespace config - +} diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index f9db9eb2f0d..eb7dad88995 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -30,3 +30,8 @@ attribute[].densepostinglistthreshold double default=0.40 attribute[].tensortype string default="" # Whether this is an imported attribute (from parent document db) or not. attribute[].imported bool default=false + +# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. +attribute[].index.hnsw.enabled bool default=false +attribute[].index.hnsw.maxlinkspernode int default=16 +attribute[].index.hnsw.neighborstoexploreatinsert int default=200 diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 8d5e7015947..33c568061fe 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,7 +7,6 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.use4443Upstream bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index c02a9018064..7816ee74fde 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -24,7 +24,7 @@ disk_operation_timeout int default=0 restart ## PERFORMANCE PARAMETERS ## Number of threads to use for each mountpoint. -num_threads int default=6 restart +num_threads int default=8 restart ## When merging, if we find more than this number of documents that exist on all ## of the same copies, send a separate apply bucket diff with these entries diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index d2f26738301..d3f37eb320e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -21,11 +21,14 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; @@ -74,6 +77,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -109,6 +113,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Orchestrator orchestrator; private final LogRetriever logRetriever; private final TesterClient testerClient; + private final Metric metric; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -118,7 +123,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye HttpProxy httpProxy, ConfigserverConfig configserverConfig, Orchestrator orchestrator, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this(tenantRepository, hostProvisionerProvider.getHostProvisioner(), infraDeployerProvider.getInfraDeployer(), @@ -129,7 +135,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye new LogRetriever(), new FileDistributionStatus(), Clock.systemUTC(), - testerClient); + testerClient, + metric); } // For testing @@ -143,7 +150,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye new ConfigserverConfig(new ConfigserverConfig.Builder()), new LogRetriever(), clock, - new TesterClient()); + new TesterClient(), + new NullMetric()); } // For testing @@ -153,7 +161,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigserverConfig configserverConfig, LogRetriever logRetriever, Clock clock, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this(tenantRepository, Optional.of(hostProvisioner), Optional.empty(), @@ -164,7 +173,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye logRetriever, new FileDistributionStatus(), clock, - testerClient); + testerClient, + metric); } private ApplicationRepository(TenantRepository tenantRepository, @@ -177,7 +187,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LogRetriever logRetriever, FileDistributionStatus fileDistributionStatus, Clock clock, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this.tenantRepository = tenantRepository; this.hostProvisioner = hostProvisioner; this.infraDeployer = infraDeployer; @@ -189,6 +200,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.fileDistributionStatus = fileDistributionStatus; this.clock = clock; this.testerClient = testerClient; + this.metric = metric; } // ---------------- Deploying ---------------------------------------------------------------- @@ -200,10 +212,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<ApplicationSet> currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); Slime deployLog = createDeployLog(); DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); - ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); - logConfigChangeActions(actions, logger); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, deployLog); + try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { + ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); + logConfigChangeActions(actions, logger); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + return new PrepareResult(sessionId, actions, deployLog); + } } public PrepareResult prepareAndActivate(Tenant tenant, long sessionId, PrepareParams prepareParams, @@ -361,15 +375,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // until the config server where the deployment happened picks it up and deletes // the local session long sessionId = activeSession.get(); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - - if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - throw new InternalServerException("Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + RemoteSession remoteSession; + try { + remoteSession = getRemoteSession(tenant, sessionId); + Transaction deleteTransaction = remoteSession.createDeleteTransaction(); + deleteTransaction.commit(); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + + if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + deleteTransaction.rollbackOrLog(); + throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId); + } + } catch (NotFoundException e) { + // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); } NestedTransaction transaction = new NestedTransaction(); @@ -417,12 +439,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<String> fileReferencesInUse = new HashSet<>(); // Intentionally skip applications that we for some reason do not find - listApplications().stream() - .map(this::getOptionalApplication) - .map(Optional::get) - .forEach(application -> fileReferencesInUse.addAll(application.getModel().fileReferences().stream() - .map(FileReference::value) - .collect(Collectors.toSet()))); + // or that we fail to get file references for (they will be retried on the next run) + for (var application : listApplications()) { + try { + Optional<Application> app = getOptionalApplication(application); + if (app.isEmpty()) continue; + fileReferencesInUse.addAll(app.get().getModel().fileReferences().stream() + .map(FileReference::value) + .collect(Collectors.toSet())); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Getting file references in use for '" + application + "' failed", e); + } + } log.log(LogLevel.DEBUG, "File references in use : " + fileReferencesInUse); // Find those on disk that are not in use @@ -468,6 +496,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); @@ -543,9 +572,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return testerClient.getLog(getTesterHostname(applicationId), getTesterPort(applicationId), after); } - // TODO: Not implemented in TesterClient yet - public HttpResponse startTests(ApplicationId applicationId, String suite, String config) { - return testerClient.startTests(getTesterHostname(applicationId), suite, config); + public HttpResponse startTests(ApplicationId applicationId, String suite, byte[] config) { + return testerClient.startTests(getTesterHostname(applicationId), getTesterPort(applicationId), suite, config); + } + + public HttpResponse isTesterReady(ApplicationId applicationId) { + return testerClient.isTesterReady(getTesterHostname(applicationId), getTesterPort(applicationId)); } private String getTesterHostname(ApplicationId applicationId) { @@ -832,4 +864,41 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye RegionName.from(configserverConfig.region())); } + /** Emits as a metric the time in millis spent while holding this timer, with deployment ID as dimensions. */ + public ActionTimer timerFor(ApplicationId id, String metricName) { + return new ActionTimer(metric, clock, id, configserverConfig.environment(), configserverConfig.region(), metricName); + } + + public static class ActionTimer implements AutoCloseable { + + private final Metric metric; + private final Clock clock; + private final ApplicationId id; + private final String environment; + private final String region; + private final String name; + private final Instant start; + + private ActionTimer(Metric metric, Clock clock, ApplicationId id, String environment, String region, String name) { + this.metric = metric; + this.clock = clock; + this.id = id; + this.environment = environment; + this.region = region; + this.name = name; + this.start = clock.instant(); + } + + @Override + public void close() { + metric.set(name, + Duration.between(start, clock.instant()).toMillis(), + metric.createContext(Map.of("applicationId", id.toFullString(), + "tenantName", id.tenant().value(), + "app", id.application().value() + "." + id.instance().value(), + "zone", environment + "." + region))); + } + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 9e81d3c0525..89d7c349d6b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -6,11 +6,14 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; @@ -22,6 +25,7 @@ import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import java.util.logging.Logger; @@ -98,19 +102,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { @Override public void prepare() { if (prepared) return; - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - - session.prepare(logger, - new PrepareParams.Builder().applicationId(session.getApplicationId()) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors( ! validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap) - .build(), - Optional.empty(), - tenant.getPath(), - clock.instant()); - this.prepared = true; + try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + + session.prepare(logger, + new PrepareParams.Builder().applicationId(session.getApplicationId()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(!validate) + .vespaVersion(version.toString()) + .isBootstrap(isBootstrap) + .build(), + Optional.empty(), + tenant.getPath(), + clock.instant()); + this.prepared = true; + } } /** Activates this. If it is not already prepared, this will call prepare first. */ @@ -119,28 +125,32 @@ public class Deployment implements com.yahoo.config.provision.Deployment { if ( ! prepared) prepare(); - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - - ApplicationId applicationId = session.getApplicationId(); - try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { - validateSessionStatus(session); - NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); - hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); - transaction.commit(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new InternalServerException("Error activating application", e); - } + try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.activateMillis")) { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - session.waitUntilActivated(timeoutBudget); + ApplicationId applicationId = session.getApplicationId(); + try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { + validateSessionStatus(session); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); + hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); + transaction.commit(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new InternalServerException("Error activating application", e); + } + + session.waitUntilActivated(timeoutBudget); - log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + - " activated successfully using " + - (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + - ". Config generation " + session.getMetaData().getGeneration() + - ". File references used: " + applicationRepository.getFileReferences(applicationId)); + log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + + " activated successfully using " + + (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + + ". Config generation " + session.getMetaData().getGeneration() + + ". File references used: " + applicationRepository.getFileReferences(applicationId)); + } } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 52d47a9398b..829a59a9598 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -130,9 +131,10 @@ public class ModelContextImpl implements ModelContext { private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; - private final Optional<TlsSecrets> tlsSecrets; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; + private final boolean useNewAthenzFilter; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -146,7 +148,7 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<TlsSecrets> tlsSecrets) { + Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -160,11 +162,13 @@ public class ModelContextImpl implements ModelContext { this.isFirstTimeDeployment = isFirstTimeDeployment; this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.tlsSecrets = tlsSecrets; + this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.useNewAthenzFilter = Flags.USE_NEW_ATHENZ_FILTER.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -208,13 +212,18 @@ public class ModelContextImpl implements ModelContext { public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override - public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } + + @Override + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return useBucketSpaceMetric; } + + @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 81eda46c257..76a022df580 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -1,30 +1,46 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.log.LogLevel; +import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; import java.net.SocketTimeoutException; +import java.net.URISyntaxException; import java.net.URL; import java.util.logging.Logger; public class SimpleHttpFetcher implements HttpFetcher { private static final Logger logger = Logger.getLogger(SimpleHttpFetcher.class.getName()); + private final CloseableHttpClient client = VespaHttpClientBuilder.create().build(); + @Override public HttpResponse get(Params params, URL url) { try { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setReadTimeout(params.readTimeoutMs); - int code = connection.getResponseCode(); - String contentType = connection.getContentType(); - InputStream inputStream = connection.getInputStream(); - StaticResponse response = new StaticResponse(code, contentType, inputStream); - return response; - } catch (SocketTimeoutException e) { + HttpGet request = new HttpGet(url.toURI()); + request.addHeader("Connection", "Close"); + request.setConfig( + RequestConfig.custom() + .setConnectTimeout(params.readTimeoutMs) + .setSocketTimeout(params.readTimeoutMs) + .build()); + try (CloseableHttpResponse response = client.execute(request)) { + HttpEntity entity = response.getEntity(); + return new StaticResponse( + response.getStatusLine().getStatusCode(), + entity.getContentType().getValue(), + EntityUtils.toString(entity)); + } + } catch (ConnectTimeoutException | SocketTimeoutException e) { String message = "Timed out after " + params.readTimeoutMs + " ms reading response from " + url; logger.log(LogLevel.WARNING, message, e); throw new RequestTimeoutException(message); @@ -32,6 +48,10 @@ public class SimpleHttpFetcher implements HttpFetcher { String message = "Failed to get response from " + url; logger.log(LogLevel.WARNING, message, e); throw new InternalServerException(message); + } catch (URISyntaxException e) { + String message = "Invalid URL: " + e.getMessage(); + logger.log(LogLevel.WARNING, message, e); + throw new InternalServerException(message, e); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 92c04bfb8e7..2d5cce2d4f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -7,8 +7,10 @@ import com.yahoo.log.LogLevel; import com.yahoo.yolean.Exceptions; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; import java.io.IOException; import java.net.URI; @@ -24,17 +26,7 @@ public class TesterClient { private static final Logger logger = Logger.getLogger(TesterClient.class.getName()); public HttpResponse getStatus(String testerHostname, int port) { - URI testerUri; - try { - testerUri = new URIBuilder() - .setScheme("https") - .setHost(testerHostname) - .setPort(port) - .setPath("/tester/v1/status") - .build(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } + URI testerUri = createURI(testerHostname, port, "/tester/v1/status"); return execute(new HttpGet(testerUri), "Failed to get tester status"); } @@ -42,11 +34,7 @@ public class TesterClient { public HttpResponse getLog(String testerHostname, int port, Long after) { URI testerUri; try { - testerUri = new URIBuilder() - .setScheme("https") - .setHost(testerHostname) - .setPort(port) - .setPath("/tester/v1/log") + testerUri = createBuilder(testerHostname, port, "/tester/v1/log") .addParameter("after", String.valueOf(after)) .build(); } catch (URISyntaxException e) { @@ -56,14 +44,22 @@ public class TesterClient { return execute(new HttpGet(testerUri), "Failed to get tester logs"); } - public HttpResponse startTests(String testerHostname, String suite, String config) { - throw new UnsupportedOperationException("Not implemented yet"); + public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) { + URI testerUri = createURI(testerHostname, port, "/tester/v1/run/" + suite); + HttpPost request = new HttpPost(testerUri); + request.setEntity(new ByteArrayEntity(config)); + + return execute(request, "Failed to start tests"); } + public HttpResponse isTesterReady(String testerHostname, int port) { + URI testerUri = createURI(testerHostname, port, "/status.html"); + + return execute(new HttpGet(testerUri), "/status.html did not return 200 OK"); + } private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { - // TODO: Change log level to DEBUG - logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString()); + logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString()); try { return new ProxyResponse(httpClient.execute(request)); } catch (IOException e) { @@ -72,4 +68,20 @@ public class TesterClient { } } + private URIBuilder createBuilder(String testerHostname, int port, String path) { + return new URIBuilder() + .setScheme("https") + .setHost(testerHostname) + .setPort(port) + .setPath(path); + } + + private URI createURI(String testerHostname, int port, String path) { + try { + return createBuilder(testerHostname, port, path).build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java index bb752f9b804..0014d66026b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index d9a5aa7055d..833c9ca4fba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.jdisc.application.UriPattern; @@ -22,6 +23,7 @@ import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import java.io.IOException; import java.time.Duration; import java.util.List; import java.util.Objects; @@ -46,6 +48,7 @@ public class ApplicationHandler extends HttpHandler { "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*", "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", "http://*/application/v2/tenant/*/application/*") @@ -138,13 +141,11 @@ public class ApplicationHandler extends HttpHandler { switch (testerCommand) { case "status": return applicationRepository.getTesterStatus(applicationId); - case "logs": + case "log": Long after = Long.valueOf(request.getProperty("after")); return applicationRepository.getTesterLog(applicationId, after); - case "startTests": - String suite = "foo"; // TODO - String config = "bar"; // TODO - return applicationRepository.startTests(applicationId, suite, config); + case "ready": + return applicationRepository.isTesterReady(applicationId); default: throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString()); } @@ -156,9 +157,19 @@ public class ApplicationHandler extends HttpHandler { @Override public HttpResponse handlePOST(HttpRequest request) { ApplicationId applicationId = getApplicationIdFromRequest(request); - if (request.getUri().getPath().endsWith("restart")) + if (request.getUri().getPath().endsWith("restart")) { return restart(request, applicationId); - throw new NotFoundException("Illegal POST request '" + request.getUri() + "': Must end with /restart"); + } else if (isTesterStartTestsRequest(request)) { + byte[] data; + try { + data = IOUtils.readBytes(request.getData(), 1024 * 1000); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read data in request " + request); + } + return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data); + } else { + throw new NotFoundException("Illegal POST request '" + request.getUri() + "'"); + } } private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { @@ -233,6 +244,12 @@ public class ApplicationHandler extends HttpHandler { request.getUri().getPath().contains("/tester"); } + private static boolean isTesterStartTestsRequest(HttpRequest request) { + System.out.println(getBindingMatch(request).groupCount()); + return getBindingMatch(request).groupCount() == 9 && + request.getUri().getPath().contains("/tester/run/"); + } + private static String getHostNameFromRequest(HttpRequest req) { BindingMatch<?> bm = getBindingMatch(req); return bm.group(7); @@ -243,6 +260,11 @@ public class ApplicationHandler extends HttpHandler { return bm.group(7); } + private static String getSuiteFromRequest(HttpRequest req) { + BindingMatch<?> bm = getBindingMatch(req); + return bm.group(8); + } + private static String getPathSuffix(HttpRequest req) { BindingMatch<?> bm = getBindingMatch(req); return bm.group(8); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 34c7dced404..197b90e322c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -6,7 +6,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index d88fae0a8ef..6366576e163 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,10 +9,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -35,12 +32,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag use4443Upstream; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource); } @Override @@ -56,10 +51,15 @@ public class LbServicesProducer implements LbServicesConfig.Producer { LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder(); apps.stream() .sorted(Comparator.comparing(ApplicationInfo::getApplicationId)) + .filter(applicationInfo -> generateRoutingConfig(applicationInfo.getApplicationId())) .forEach(applicationInfo -> tb.applications(createLbAppIdKey(applicationInfo.getApplicationId()), getAppConfig(applicationInfo))); return tb; } + private boolean generateRoutingConfig(ApplicationId applicationId) { + return ( ! applicationId.instance().isTester()); + } + private String createLbAppIdKey(ApplicationId applicationId) { return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value(); } @@ -67,8 +67,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); - ab.use4443Upstream( - use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); @@ -92,10 +90,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); hb.hostname(hostInfo.getHostname()); - hostInfo.getServices().stream() - .forEach(serviceInfo -> { - hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo)); - }); + hostInfo.getServices().forEach(serviceInfo -> hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo))); return hb; } @@ -114,12 +109,11 @@ public class LbServicesProducer implements LbServicesConfig.Producer { filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList())) .endpointaliases(endpointAliases) .index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999"))); - serviceInfo.getPorts().stream() - .forEach(portInfo -> { - LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() - .number(portInfo.getPort()) - .tags(Joiner.on(" ").join(portInfo.getTags())); - sb.ports(pb); + serviceInfo.getPorts().forEach(portInfo -> { + LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() + .number(portInfo.getPort()) + .tags(Joiner.on(" ").join(portInfo.getTags())); + sb.ports(pb); }); return sb; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index bc6419f230f..a2fc2bfd6a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -27,8 +27,9 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -135,7 +136,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating flagSource, - new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId)); + new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) + .readEndpointCertificateMetadata(applicationId) + .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 87e9fa287a4..249c8639ea9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.monitoring; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -14,14 +16,17 @@ import com.yahoo.statistics.Counter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** - * Statistics for server. The statistics framework takes care of logging. + * Metrics for config server. The statistics framework takes care of logging. * * @author Harald Musum - * @since 4.2 */ -public class Metrics extends TimerTask implements MetricUpdaterFactory { +public class Metrics extends AbstractComponent implements MetricUpdaterFactory, Runnable { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName()); private static final String METRIC_REQUESTS = getMetricName("requests"); @@ -33,23 +38,34 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { private final Counter failedRequests; private final Counter procTimeCounter; private final Metric metric; - private final ZKMetricUpdater zkMetricUpdater; + private final Optional<ZKMetricUpdater> zkMetricUpdater; // TODO The map is the key for now private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>(); - private final Timer timer = new Timer(); + private final Optional<ScheduledExecutorService> executorService; @Inject public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, ZookeeperServerConfig zkServerConfig) { + this(metric, statistics, healthMonitorConfig, zkServerConfig, true); + } + + private Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, + ZookeeperServerConfig zkServerConfig, boolean createZkMetricUpdater) { this.metric = metric; requests = createCounter(METRIC_REQUESTS, statistics); failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics); procTimeCounter = createCounter("procTime", statistics); - log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); - long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); - timer.scheduleAtFixedRate(this, 20000, intervalMs); - zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs); + if (createZkMetricUpdater) { + log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); + long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); + executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics"))); + executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS); + zkMetricUpdater = Optional.of(new ZKMetricUpdater(zkServerConfig, 19500, intervalMs)); + } else { + executorService = Optional.empty(); + zkMetricUpdater = Optional.empty(); + } } public static Metrics createTestMetrics() { @@ -58,14 +74,13 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); builder.snapshot_interval(60.0); ZookeeperServerConfig.Builder zkBuilder = new ZookeeperServerConfig.Builder().myid(1); - return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder)); + return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder), false); } private Counter createCounter(String name, Statistics statistics) { return new Counter(name, statistics, false); } - void incrementRequests(Metric.Context metricContext) { requests.increment(1); metric.add(METRIC_REQUESTS, 1, metricContext); @@ -126,8 +141,12 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { } } setRegularMetrics(); - zkMetricUpdater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null)); - timer.purge(); + zkMetricUpdater.ifPresent(updater -> updater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null))); + } + + public void deconstruct() { + executorService.ifPresent(ExecutorService::shutdown); + zkMetricUpdater.ifPresent(ZKMetricUpdater::shutdown); } private void setRegularMetrics() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index b2813be5456..62da6fcffbe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.monitoring; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -17,6 +18,8 @@ import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; @@ -26,7 +29,7 @@ import java.util.regex.Pattern; import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName; -public class ZKMetricUpdater extends TimerTask { +public class ZKMetricUpdater implements Runnable { private static final Logger log = Logger.getLogger(ZKMetricUpdater.class.getName()); public static final String METRIC_ZK_ZNODES = getMetricName("zkZNodes"); @@ -35,19 +38,19 @@ public class ZKMetricUpdater extends TimerTask { public static final String METRIC_ZK_CONNECTIONS = getMetricName("zkConnections"); public static final String METRIC_ZK_OUTSTANDING_REQUESTS = getMetricName("zkOutstandingRequests"); - private final int CONNECTION_TIMEOUT_MS = 500; - private final int WRITE_TIMEOUT_MS = 250; - private final int READ_TIMEOUT_MS = 500; + private static final int CONNECTION_TIMEOUT_MS = 500; + private static final int WRITE_TIMEOUT_MS = 250; + private static final int READ_TIMEOUT_MS = 500; private AtomicReference<Map<String, Long>> zkMetrics = new AtomicReference<>(new HashMap<>()); - private final Timer timer = new Timer(); + private final ScheduledExecutorService executorService; private final int zkPort; public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) { this.zkPort = zkServerConfig.clientPort(); - if (intervalMS > 0) { - timer.scheduleAtFixedRate(this, delayMS, intervalMS); - } + if (intervalMS <= 0 ) throw new IllegalArgumentException("interval must be positive, was " + intervalMS + " ms"); + this.executorService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("zkmetricupdater")); + this.executorService.scheduleAtFixedRate(this, delayMS, intervalMS, TimeUnit.MILLISECONDS); } private void setMetricAttribute(String attribute, long value, Map<String, Long> data) { @@ -74,7 +77,10 @@ public class ZKMetricUpdater extends TimerTask { public void run() { Optional<String> report = retrieveReport(); report.ifPresent(this::parseReport); - timer.purge(); + } + + public void shutdown() { + executorService.shutdown(); } private Optional<String> retrieveReport() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 67aa49ea0b9..48580dbc6f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -167,7 +167,7 @@ class GetConfigProcessor implements Runnable { log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); - ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5); + ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5); request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5()); respond(request); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java index dcbc21e536c..5235a2bcadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java @@ -29,7 +29,7 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory { String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes())); - return new SlimeConfigResponse(compressed, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(compressed, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java index 91809f90a70..bd0b117c3db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java @@ -25,7 +25,7 @@ public class UncompressedConfigResponseFactory implements ConfigResponseFactory Utf8Array rawPayload = payload.toUtf8Array(true); String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); - return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(rawPayload, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index ab3e0e863ce..49f59773bca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -3,14 +3,16 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import java.time.Clock; import java.time.Duration; @@ -32,6 +34,7 @@ public final class PrepareParams { static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; + static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -42,10 +45,12 @@ public final class PrepareParams { private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) { + List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, + Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -55,6 +60,7 @@ public final class PrepareParams { this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; this.tlsSecretsKeyName = tlsSecretsKeyName; + this.endpointCertificateMetadata = endpointCertificateMetadata; } public static class Builder { @@ -68,6 +74,7 @@ public final class PrepareParams { private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = List.of(); private Optional<String> tlsSecretsKeyName = Optional.empty(); + private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); public Builder() { } @@ -128,9 +135,16 @@ public final class PrepareParams { return this; } + public Builder endpointCertificateMetadata(String serialized) { + if(serialized == null) return this; + Slime slime = SlimeUtils.jsonToSlime(serialized); + endpointCertificateMetadata = Optional.of(EndpointCertificateMetadataSerializer.fromSlime(slime.get())); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName); + verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata); } } @@ -144,6 +158,7 @@ public final class PrepareParams { .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) + .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .build(); } @@ -200,4 +215,8 @@ public final class PrepareParams { public Optional<String> tlsSecretsKeyName() { return tlsSecretsKeyName; } + + public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { + return endpointCertificateMetadata; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 171eab35507..0115876ded9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -12,8 +12,9 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -33,7 +34,9 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import org.xml.sax.SAXException; @@ -113,7 +116,7 @@ public class SessionPreparer { preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); - preparation.writeTlsZK(); + preparation.writeEndpointCertificateMetadataZK(); preparation.writeContainerEndpointsZK(); preparation.distribute(); } @@ -142,8 +145,10 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpoints; final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; - private final TlsSecretsKeys tlsSecretsKeys; - private final Optional<TlsSecrets> tlsSecrets; + private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private final EndpointCertificateRetriever endpointCertificateRetriever; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -162,8 +167,16 @@ public class SessionPreparer { this.applicationId = params.getApplicationId(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); - this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); + this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + this.endpointCertificateMetadata = params.endpointCertificateMetadata() + .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); + + endpointCertificateSecrets = endpointCertificateMetadata + .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + this.endpointsSet = getEndpoints(params.containerEndpoints()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), @@ -178,7 +191,7 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - tlsSecrets); + endpointCertificateSecrets); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -233,9 +246,10 @@ public class SessionPreparer { checkTimeout("write state to zookeeper"); } - void writeTlsZK() { - tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null)); - checkTimeout("write tlsSecretsKey to zookeeper"); + void writeEndpointCertificateMetadataZK() { + endpointCertificateMetadata.ifPresent(metadata -> + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, metadata)); + checkTimeout("write endpoint certificate metadata to zookeeper"); } void writeContainerEndpointsZK() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java index 65ebb38c2d0..97179e5234b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java new file mode 100644 index 00000000000..f3240a62133 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +/** + * (de)serializes endpoint certificate metadata + * + * @author andreer + */ +public class EndpointCertificateMetadataSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final static String keyNameField = "keyName"; + private final static String certNameField = "certName"; + private final static String versionField = "version"; + + public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) { + object.setString(keyNameField, metadata.keyName()); + object.setString(certNameField, metadata.certName()); + object.setLong(versionField, metadata.version()); + } + + public static EndpointCertificateMetadata fromSlime(Inspector inspector) { + switch (inspector.type()) { + case STRING: // TODO: Remove once all are transmitted and stored as JSON + return new EndpointCertificateMetadata( + inspector.asString() + "-key", + inspector.asString() + "-cert", + 0 + ); + case OBJECT: + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()) + ); + + default: + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + } + } + + public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { + return fromSlime(new Slime().setString(tlsSecretsKeys)); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java new file mode 100644 index 00000000000..8e51ac424f9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * Stores the endpoint certificate metadata for an application. + * This metadata is then used to retrieve the actual secrets from {@link EndpointCertificateRetriever}. + * + * @author andreer + */ +public class EndpointCertificateMetadataStore { + + private final Path path; + private final Curator curator; + + public EndpointCertificateMetadataStore(Curator curator, Path tenantPath) { + this.curator = curator; + this.path = tenantPath.append("tlsSecretsKeys/"); + } + + /** Reads the endpoint certificate metadata from ZooKeeper, if it exists */ + public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(endpointCertificateMetadataPathOf(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + Slime slime = SlimeUtils.jsonToSlime(data.get()); + EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); + return Optional.of(endpointCertificateMetadata); + } catch (Exception e) { + throw new RuntimeException("Error reading TLS secret key of " + application, e); + } + } + + /** Writes the endpoint certificate metadata to ZooKeeper */ + public void writeEndpointCertificateMetadata(ApplicationId application, EndpointCertificateMetadata endpointCertificateMetadata) { + try { + Slime slime = new Slime(); + EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); + curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + /** Returns a transaction which deletes these tls secrets key if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); + } + + /** Returns the path storing the tls secrets key for an application */ + private Path endpointCertificateMetadataPathOf(ApplicationId application) { + return path.append(application.serializedForm()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java new file mode 100644 index 00000000000..5f40e5e1411 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; + +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.cert.X509Certificate; +import java.util.Optional; + +/** + * Used to retrieve actual endpoint certificate/key from secret store. + * + * @author andreer + */ +public class EndpointCertificateRetriever { + + private final SecretStore secretStore; + + public EndpointCertificateRetriever(SecretStore secretStore) { + this.secretStore = secretStore; + } + + public Optional<EndpointCertificateSecrets> readEndpointCertificateSecrets(EndpointCertificateMetadata metadata) { + return Optional.of(readFromSecretStore(metadata)); + } + + private EndpointCertificateSecrets readFromSecretStore(EndpointCertificateMetadata endpointCertificateMetadata) { + try { + String cert = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); + String key = secretStore.getSecret(endpointCertificateMetadata.keyName(), endpointCertificateMetadata.version()); + + verifyKeyMatchesCertificate(endpointCertificateMetadata, cert, key); + + return new EndpointCertificateSecrets(cert, key); + } catch (RuntimeException e) { + // Assume not ready yet + return EndpointCertificateSecrets.MISSING; + } + } + + private void verifyKeyMatchesCertificate(EndpointCertificateMetadata endpointCertificateMetadata, String cert, String key) { + X509Certificate x509Certificate = X509CertificateUtils.fromPem(cert); + + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(key); + PublicKey publicKey = x509Certificate.getPublicKey(); + + if(!X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, publicKey)) { + throw new IllegalArgumentException("Failed to retrieve endpoint secrets: Certificate and key data do not match for " + endpointCertificateMetadata); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java deleted file mode 100644 index da6fc490da9..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.TlsSecrets; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.path.Path; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.transaction.CuratorOperations; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; - -import java.util.Optional; - -/** - * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper. - * - * @author andreer - */ -public class TlsSecretsKeys { - - private final Path path; - private final SecretStore secretStore; - private final Curator curator; - - public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) { - this.curator = curator; - this.path = tenantPath.append("tlsSecretsKeys/"); - this.secretStore = secretStore; - } - - public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) { - try { - Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application)); - if (data.isEmpty() || data.get().length == 0) return Optional.empty(); - - Slime slime = SlimeUtils.jsonToSlime(data.get()); - final var inspector = slime.get(); - - switch (inspector.type()) { - case STRING: // TODO: Remove once all are stored as JSON - return readFromSecretStore(Optional.ofNullable(inspector.asString())); - case OBJECT: - var tlsSecretsInfo = new TlsSecretsMetadata(); - tlsSecretsInfo.certName = inspector.field("certName").asString(); - tlsSecretsInfo.keyName = inspector.field("keyName").asString(); - tlsSecretsInfo.version = Math.toIntExact(inspector.field("version").asLong()); - return Optional.of(readFromSecretStore(tlsSecretsInfo)); - default: - throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); - } - } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); - } - } - - public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) { - if (tlsSecretsKey == null) return; - writeTlsSecretsAsString(application, tlsSecretsKey); - } - - private void writeTlsSecretsAsString(ApplicationId application, String tlsSecretsKey) { - try { - Slime slime = new Slime(); - slime.setString(tlsSecretsKey); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - void writeTlsSecretsMetadata(ApplicationId application, TlsSecretsMetadata tlsSecretsMetadata) { - try { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString(TlsSecretsMetadata.certNameField, tlsSecretsMetadata.certName); - cursor.setString(TlsSecretsMetadata.keyNameField, tlsSecretsMetadata.keyName); - cursor.setLong(TlsSecretsMetadata.versionField, tlsSecretsMetadata.version); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) { - if (secretKeyname == null || secretKeyname.isEmpty()) { - return readTlsSecretsKeyFromZookeeper(applicationId); - } - return readFromSecretStore(secretKeyname); - } - - private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) { - if (secretKeyname.isEmpty()) return Optional.empty(); - try { - String cert = secretStore.getSecret(secretKeyname.get() + "-cert"); - String key = secretStore.getSecret(secretKeyname.get() + "-key"); - return Optional.of(new TlsSecrets(cert, key)); - } catch (RuntimeException e) { - // Assume not ready yet - return Optional.of(TlsSecrets.MISSING); - } - } - - private TlsSecrets readFromSecretStore(TlsSecretsMetadata tlsSecretsMetadata) { - try { - String cert = secretStore.getSecret(tlsSecretsMetadata.certName, tlsSecretsMetadata.version); - String key = secretStore.getSecret(tlsSecretsMetadata.keyName, tlsSecretsMetadata.version); - return new TlsSecrets(cert, key); - } catch (RuntimeException e) { - // Assume not ready yet - return TlsSecrets.MISSING; - } - } - - /** Returns a transaction which deletes these tls secrets key if they exist */ - public CuratorTransaction delete(ApplicationId application) { - if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator); - return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator); - } - - /** Returns the path storing the tls secrets key for an application */ - private Path tlsSecretsKeyOf(ApplicationId application) { - return path.append(application.serializedForm()); - } - - static class TlsSecretsMetadata { - final static String keyNameField = "keyName"; - final static String certNameField = "certName"; - final static String versionField = "version"; - String keyName; - String certName; - int version; - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index fb745bbb76b..0e076d60d52 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -12,7 +12,9 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.OrchestratorMock; @@ -40,6 +42,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -266,13 +270,22 @@ public class ApplicationRepositoryTest { } { + PrepareResult prepareResult = deployApp(testApp); try { - deployApp(testApp); applicationRepository.delete(applicationId(), Duration.ZERO); fail("Should have gotten an exception"); } catch (InternalServerException e) { - assertEquals("Session 5 was not deleted (waited PT0S)", e.getMessage()); + assertEquals("test1.testapp was not deleted (waited PT0S), session " + prepareResult.sessionId(), e.getMessage()); } + + // No active session or remote session (deleted in step above), but an exception was thrown above + // A new delete should cleanup and be successful + LocalSession activeSession = applicationRepository.getActiveSession(applicationId()); + assertNull(activeSession); + Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); + assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + + assertTrue(applicationRepository.delete(applicationId())); } } @@ -317,6 +330,28 @@ public class ApplicationRepositoryTest { assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0))); } + @Test + public void testMetrics() { + MockMetric actual = new MockMetric(); + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new MockLogRetriever(), + new ManualClock(), + new MockTesterClient(), + actual); + deployApp(testAppLogServerWithContainer); + Map<String, ?> context = Map.of("applicationId", "test1.testapp.default", + "tenantName", "test1", + "app", "testapp.default", + "zone", "prod.default"); + MockMetric expected = new MockMetric(); + expected.set("deployment.prepareMillis", 0L, expected.createContext(context)); + expected.set("deployment.activateMillis", 0L, expected.createContext(context)); + assertEquals(expected.values, actual.values); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, @@ -324,7 +359,8 @@ public class ApplicationRepositoryTest { new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), clock, - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); } private PrepareResult prepareAndActivateApp(File application) throws IOException { @@ -360,4 +396,39 @@ public class ApplicationRepositoryTest { return applicationRepository.getMetadataFromSession(tenant, sessionId); } + + /** Stores all added or set values for each metric and context. */ + static class MockMetric implements Metric { + + final Map<String, Map<Map<String, ?>, Number>> values = new HashMap<>(); + + @Override + public void set(String key, Number val, Metric.Context ctx) { + values.putIfAbsent(key, new HashMap<>()); + values.get(key).put(((Context) ctx).point, val); + } + + @Override + public void add(String key, Number val, Metric.Context ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + return new Context(properties); + } + + + private static class Context implements Metric.Context { + + private final Map<String, ?> point; + + public Context(Map<String, ?> point) { + this.point = Map.copyOf(point); + } + + } + + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java index 8a77b53875e..12f48778144 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -7,22 +7,26 @@ import java.util.HashMap; import java.util.Map; public class MockSecretStore implements SecretStore { - Map<String, String> secrets = new HashMap<>(); + Map<String, Map<Integer, String>> secrets = new HashMap<>(); @Override public String getSecret(String key) { if(secrets.containsKey(key)) - return secrets.get(key); + return secrets.get(key).get(0); throw new RuntimeException("Key not found: " + key); } @Override public String getSecret(String key, int version) { - return getSecret(key); + return secrets.get(key).get(version); + } + + public void put(String key, int version, String value) { + secrets.computeIfAbsent(key, k -> new HashMap<>()).put(version, value); } public void put(String key, String value) { - secrets.put(key, value); + put(key, 0, value); } public void remove(String key) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java index 7cb878a5f2c..c835bb29ec0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java @@ -15,38 +15,40 @@ public class MockTesterClient extends TesterClient { @Override public HttpResponse getStatus(String testerHostname, int port) { - return new MockStatusResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); + } + }; } @Override public HttpResponse getLog(String testerHostname, int port, Long after) { - return new MockLogResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log".getBytes(StandardCharsets.UTF_8)); + } + }; } - private static class MockStatusResponse extends HttpResponse { - - private MockStatusResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); - } - + @Override + public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) { } + }; } - private static class MockLogResponse extends HttpResponse { - - private MockLogResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("log".getBytes(StandardCharsets.UTF_8)); - } - + @Override + public HttpResponse isTesterReady(String testerHostname, int port) { + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("{ \"message\": \"OK\" } ".getBytes(StandardCharsets.UTF_8)); + } + }; } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java index f2ee7815df3..9a18570db2d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java @@ -27,7 +27,6 @@ public class ServerCacheTest { private static String configMd5 = "mymd5"; private static String configMd5_2 = "mymd5_2"; private static ConfigDefinition def = new ConfigDefinition("mypayload", new String[0]); - private static ConfigDefinition def_2 = new ConfigDefinition("otherpayload", new String[0]); private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar"); private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz"); @@ -49,9 +48,9 @@ public class ServerCacheTest { cache = new ServerCache(new TestConfigDefinitionRepo(), userConfigDefinitionRepo); - cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def_2.getCNode(), 2, false, configMd5_2), configMd5_2); + cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5_2), configMd5_2); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java index 405fff3e190..ad910c2afc2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -4,14 +4,15 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ModelConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; @@ -33,12 +34,14 @@ import org.junit.Before; import org.junit.Test; import org.xml.sax.SAXException; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.util.List; +import java.nio.charset.StandardCharsets; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -111,22 +114,27 @@ public class ApplicationTest { } @Test - public void require_that_build_config_can_be_resolved() { - List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload(); - assertTrue(payload.get(1).contains("host")); + public void require_that_build_config_can_be_resolved() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, + ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\"")); } @Test - public void require_that_non_existent_fields_in_schema_is_skipped() { + public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException { // Ask for config without schema and check that we get correct default value back - List<String> payload = handler.resolveConfig(createSimpleConfigRequest()).getLegacyPayload(); - assertThat(payload.get(0), is("boolval false")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createSimpleConfigRequest()).serialize(baos, CompressionType.UNCOMPRESSED);; + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(StandardCharsets.UTF_8)); // Ask for config with wrong schema String[] schema = new String[1]; schema[0] = "boolval bool default=true"; // changed to be true, original is false - payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload(); - assertThat(payload.size(), is(1)); - assertThat(payload.get(0), is("boolval true")); + baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset())); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 71ae6955e56..1f034a92cbb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import org.junit.Before; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java index beb8abb7be6..837cbd4605a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java @@ -6,11 +6,15 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; +import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -22,7 +26,7 @@ import java.util.function.Function; */ public class OrchestratorMock implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> hostInfos = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -32,12 +36,13 @@ public class OrchestratorMock implements Orchestrator { @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = hostInfos.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostStatus>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)); + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -45,12 +50,12 @@ public class OrchestratorMock implements Orchestrator { @Override public void resume(HostName hostName) { - suspendedHosts.remove(hostName); + hostInfos.remove(hostName); } @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 9a15d6528ee..32b704dd551 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -27,6 +27,7 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockTesterClient; @@ -137,7 +138,8 @@ public class DeployTester { configserverConfig, new LogRetriever(), clock, - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); } public Tenant tenant() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java index 9a371639f52..ef0a5f6113d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java @@ -2,34 +2,25 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.ConfigResponse; - import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import org.junit.Test; import java.io.IOException; -import java.io.StringReader; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class HttpConfigResponseTest { @Test public void require_that_response_is_created_from_config() throws IOException { final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); - ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"); + ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"); HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 3ae98c1b8f2..089b662b797 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -2,28 +2,27 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.config.provision.ApplicationId; - +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import org.junit.Before; import org.junit.Test; + import java.io.IOException; -import java.io.StringReader; import java.util.Collections; import java.util.HashSet; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST; +import static com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -48,8 +47,7 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -75,16 +73,10 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 9e8d483cd86..67677822317 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; @@ -31,10 +32,13 @@ import org.junit.Before; import org.junit.Test; import javax.ws.rs.client.Client; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.time.Clock; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; @@ -79,7 +83,8 @@ public class ApplicationHandlerTest { new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), Clock.systemUTC(), - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); @@ -177,7 +182,8 @@ public class ApplicationHandlerTest { mockHttpProxy, new ConfigserverConfig(new ConfigserverConfig.Builder()), new OrchestratorMock(), - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); @@ -238,6 +244,42 @@ public class ApplicationHandlerTest { assertEquals("OK", baos.toString()); } + @Test + public void testTesterGetLog() throws IOException { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/log?after=1234"; + ApplicationHandler mockHandler = createApplicationHandler(); + + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + assertEquals(200, response.getStatus()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + assertEquals("log", baos.toString()); + } + + @Test + public void testTesterStartTests() { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/run/staging-test"; + ApplicationHandler mockHandler = createApplicationHandler(); + + InputStream requestData = new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); + HttpRequest testRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.POST, requestData); + HttpResponse response = mockHandler.handle(testRequest); + assertEquals(200, response.getStatus()); + } + + @Test + public void testTesterReady() { + applicationRepository.deploy(testApp, prepareParams(applicationId)); + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/ready"; + ApplicationHandler mockHandler = createApplicationHandler(); + HttpRequest testRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET); + HttpResponse response = mockHandler.handle(testRequest); + assertEquals(200, response.getStatus()); + } + private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default"; deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index fb09aa99039..b72785876bc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -1,37 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; -import java.io.IOException; -import java.io.StringReader; -import java.util.Collections; -import java.util.HashSet; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.junit.Before; -import org.junit.Test; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; +import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class HttpGetConfigHandlerTest { @@ -58,9 +57,8 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -71,11 +69,10 @@ public class HttpGetConfigHandlerTest { "/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid"; final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder() .tenant(tenant) .applicationName("myapplication").instanceName("myinstance").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -121,18 +118,11 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java index 0894e38ce09..3f67d8e2cac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java @@ -14,13 +14,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** @@ -29,11 +31,13 @@ import static org.junit.Assert.*; public class ClusterMetricsRetrieverTest { @Rule - public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); @Test public void testMetricAggregation() throws IOException { - List<URI> hosts = List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"), URI.create("http://localhost:8080/3")); + List<URI> hosts = Stream.of(1, 2, 3) + .map(item -> URI.create("http://localhost:" + wireMock.port() + "/" + item)) + .collect(Collectors.toList()); stubFor(get(urlEqualTo("/1")) .willReturn(aResponse() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 9a7cb72804f..f75d11a145f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; @@ -53,7 +54,7 @@ public class LbServicesProducerTest { private static final Set<ContainerEndpoint> endpoints = Set.of( new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2")) ); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final boolean useGlobalServiceId; @Parameterized.Parameters @@ -141,6 +142,23 @@ public class LbServicesProducerTest { assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); } + + @Test + public void testRoutingConfigForTesterApplication() throws IOException, SAXException { + assumeFalse(useGlobalServiceId); + + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); + LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); + LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); + assertThat(services.servicealiases().size(), is(1)); + assertThat(services.endpointaliases().size(), is(2)); + + // No config for tester application + assertNull(getLbServicesConfig(Zone.defaultZone(), testModel) + .tenants("foo") + .applications("baz:prod:default:custom-t")); + } + private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) { Map<TenantName, Set<ApplicationInfo>> randomizedApplications = new LinkedHashMap<>(); List<TenantName> keys = new ArrayList<>(testModel.keySet()); @@ -166,7 +184,7 @@ public class LbServicesProducerTest { Set<ApplicationInfo> aMap = new LinkedHashSet<>(); ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); - ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build(); + ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").instanceName("custom-t").build(); // tester app aMap.add(createApplication(fooApp, deploystateBuilder)); aMap.add(createApplication(barApp, deploystateBuilder)); aMap.add(createApplication(bazApp, deploystateBuilder)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java index ed089109759..607a2dca6c6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java @@ -57,12 +57,14 @@ public class ZKMetricUpdaterTest { assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_LATENCY_MAX), equalTo(1234L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_OUTSTANDING_REQUESTS), equalTo(12L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_ZNODES), equalTo(4L)); + + updater.shutdown(); } private ZKMetricUpdater buildUpdater() { ZookeeperServerConfig zkServerConfig = new ZookeeperServerConfig( new ZookeeperServerConfig.Builder().clientPort(serverPort).myid(12345)); - return new ZKMetricUpdater(zkServerConfig, 0, -1); + return new ZKMetricUpdater(zkServerConfig, 0, 100000); } private void setupTcpServer(Supplier<String> reportProvider) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 086dfa5d0d3..0b33de2a42c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -1,18 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; -import com.google.common.base.Joiner; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadApplier; @@ -24,12 +21,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.util.ConfigUtils; - import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; @@ -37,11 +33,14 @@ import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import java.io.IOException; -import java.io.StringReader; import java.util.Optional; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -161,11 +160,7 @@ public class RpcServerTest { builder.intval(123); SimpletypesConfig responseConfig = new SimpletypesConfig(builder); ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); - InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, - new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))) - .getTree(); return SlimeConfigResponse.fromConfigPayload(responsePayload, - targetDef, 3L, true, /* internalRedeploy */ ConfigUtils.getMd5(responsePayload)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a099db5ebe8..40115170b69 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -22,6 +22,11 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockReloadHandler; @@ -37,7 +42,8 @@ import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -46,9 +52,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.security.auth.x500.X500Principal; import java.io.File; import java.io.IOException; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -73,6 +84,9 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; @@ -231,15 +245,37 @@ public class SessionPreparerTest { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - secretStore.put(tlskey+"-cert", "CERT"); - secretStore.put(tlskey+"-key", "KEY"); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); // Read from zk and verify cert and key are available - Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void require_that_endpoint_certificate_metadata_is_written() throws IOException { + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); + secretStore.put("vespa.tlskeys.tenant1--app1-cert", 7, X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", 7, KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + // Read from zk and verify cert and key are available + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); } @Test(expected = CertificateNotReadyException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java new file mode 100644 index 00000000000..d71eab25ce3 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java @@ -0,0 +1,90 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class EndpointCertificateMetadataStoreTest { + + private static final Path tenantPath = Path.createRoot(); + private static final Path endpointCertificateMetadataPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); + private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), + ApplicationName.from("test"), InstanceName.defaultName()); + + private MockCurator curator; + private MockSecretStore secretStore = new MockSecretStore(); + private EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private EndpointCertificateRetriever endpointCertificateRetriever; + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); + + @Before + public void setUp() { + curator = new MockCurator(); + endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + } + + @Test + public void reads_string_format() { + curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); + + // Read from zk and verify cert and key are available + var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void reads_object_format() { + curator.set(endpointCertificateMetadataPath, + "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" + .getBytes()); + + // Read from zk and verify cert and key are available + var secrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(secrets.isPresent()); + assertTrue(secrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(secrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void can_write_object_format() { + var endpointCertificateMetadata = new EndpointCertificateMetadata("key-name", "cert-name", 1); + + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, endpointCertificateMetadata); + + assertEquals("{\"keyName\":\"key-name\",\"certName\":\"cert-name\",\"version\":1}", + new String(curator.getData(endpointCertificateMetadataPath).orElseThrow())); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java deleted file mode 100644 index c71c7b8e040..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.MockSecretStore; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TlsSecretsKeysTest { - - private static final Path tenantPath = Path.createRoot(); - private static final Path tlsSecretsKeysPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); - private static final String tlskey = "vespa.tlskeys.tenant1--app1"; - private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), - ApplicationName.from("test"), InstanceName.defaultName()); - - private MockCurator curator; - private MockSecretStore secretStore = new MockSecretStore(); - private TlsSecretsKeys tlsSecretsKeys; - - @Before - public void setUp() { - curator = new MockCurator(); - tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - secretStore.put(tlskey + "-cert", "CERT"); - secretStore.put(tlskey + "-key", "KEY"); - } - - @Test - public void reads_string_format() { - curator.set(tlsSecretsKeysPath, ('"' + tlskey + '"').getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void reads_object_format() { - curator.set(tlsSecretsKeysPath, - "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" - .getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void can_write_object_format() { - var tlsSecretsMetadata = new TlsSecretsKeys.TlsSecretsMetadata(); - tlsSecretsMetadata.certName = "cert-name"; - tlsSecretsMetadata.keyName = "key-name"; - tlsSecretsMetadata.version = 1; - - tlsSecretsKeys.writeTlsSecretsMetadata(applicationId, tlsSecretsMetadata); - - assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}", - new String(curator.getData(tlsSecretsKeysPath).get())); - } -} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index b943c03f48f..d8085cc808b 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -2,6 +2,7 @@ package com.yahoo.container.logging; import com.yahoo.collections.ListMap; +import com.yahoo.yolean.trace.TraceNode; import javax.security.auth.x500.X500Principal; import java.net.InetAddress; @@ -69,6 +70,7 @@ public class AccessLogEntry { private X500Principal sslPrincipal; private String rawPath; private String rawQuery; + private TraceNode traceNode; private ListMap<String,String> keyValues=null; @@ -452,6 +454,18 @@ public class AccessLogEntry { } } + public void setTrace(TraceNode traceNode) { + synchronized (monitor) { + requireNull(this.traceNode); + this.traceNode = traceNode; + } + } + public TraceNode getTrace() { + synchronized (monitor) { + return traceNode; + } + } + @Override public String toString() { synchronized (monitor) { @@ -481,6 +495,7 @@ public class AccessLogEntry { ", sslPrincipal=" + sslPrincipal + ", rawPath='" + rawPath + '\'' + ", rawQuery='" + rawQuery + '\'' + + ", trace='" + traceNode + '\'' + ", keyValues=" + keyValues + '}'; } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 556b97ced62..ae794e5b60a 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.yolean.trace.TraceNode; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -55,8 +56,7 @@ public class JSONFormatter { generator.writeStartObject(); generator.writeStringField("ip", accessLogEntry.getIpV4Address()); generator.writeNumberField("time", toTimestampInSeconds(accessLogEntry.getTimeStampMillis())); - generator.writeNumberField("duration", - durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); + generator.writeNumberField("duration", durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis())); generator.writeNumberField("responsesize", accessLogEntry.getReturnedContentSize()); generator.writeNumberField("code", accessLogEntry.getStatusCode()); generator.writeStringField("method", accessLogEntry.getHttpMethod()); @@ -95,6 +95,15 @@ public class JSONFormatter { } } + TraceNode trace = accessLogEntry.getTrace(); + if (trace != null) { + long timestamp = trace.timestamp(); + if (timestamp == 0L) { + timestamp = accessLogEntry.getTimeStampMillis(); + } + trace.accept(new TraceRenderer(generator, timestamp)); + } + // Only add search sub block of this is a search request if (isSearchRequest(accessLogEntry)) { generator.writeObjectFieldStart("search"); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java new file mode 100644 index 00000000000..295786aa15d --- /dev/null +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java @@ -0,0 +1,185 @@ +package com.yahoo.container.logging; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.simple.JsonRender; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; +import com.fasterxml.jackson.core.JsonGenerator; + +import java.io.IOException; + +public class TraceRenderer extends TraceVisitor { + private static final String TRACE_CHILDREN = "children"; + private static final String TRACE_MESSAGE = "message"; + private static final String TRACE_TIMESTAMP = "timestamp"; + private static final String TRACE = "trace"; + + private final long basetime; + private final JsonGenerator generator; + private final FieldConsumer fieldConsumer; + private boolean hasFieldName = false; + int emittedChildNesting = 0; + int currentChildNesting = 0; + private boolean insideOpenObject = false; + + public interface FieldConsumer { + void accept(Object object) throws IOException; + } + + private static class Consumer implements FieldConsumer { + private final JsonGenerator generator; + + Consumer(JsonGenerator generator) { + this.generator = generator; + } + + @Override + public void accept(Object object) throws IOException { + if (object instanceof Inspectable) { + renderInspectorDirect(((Inspectable) object).inspect()); + } else { + generator.writeObject(object); + } + } + private void renderInspectorDirect(Inspector data) throws IOException { + StringBuilder intermediate = new StringBuilder(); + JsonRender.render(data, intermediate, true); + generator.writeRawValue(intermediate.toString()); + } + } + + TraceRenderer(JsonGenerator generator, long basetime) { + this(generator, new Consumer(generator), basetime); + } + public TraceRenderer(JsonGenerator generator, FieldConsumer consumer, long basetime) { + this.generator = generator; + this.fieldConsumer = consumer; + this.basetime = basetime; + } + + @Override + public void entering(TraceNode node) { + ++currentChildNesting; + } + + @Override + public void leaving(TraceNode node) { + conditionalEndObject(); + if (currentChildNesting == emittedChildNesting) { + try { + generator.writeEndArray(); + generator.writeEndObject(); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + --emittedChildNesting; + } + --currentChildNesting; + } + + @Override + public void visit(TraceNode node) { + try { + doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext()); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + + private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { + boolean dirty = false; + if (timestamp != 0L) { + header(); + generator.writeStartObject(); + generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime); + dirty = true; + } + if (payload != null) { + if (!dirty) { + header(); + generator.writeStartObject(); + } + generator.writeFieldName(TRACE_MESSAGE); + fieldConsumer.accept(payload); + dirty = true; + } + if (dirty) { + if (!hasChildren) { + generator.writeEndObject(); + } else { + setInsideOpenObject(true); + } + } + } + private void header() { + fieldName(); + for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) { + startChildArray(); + } + emittedChildNesting = currentChildNesting; + } + + private void startChildArray() { + try { + conditionalStartObject(); + generator.writeArrayFieldStart(TRACE_CHILDREN); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + + private void conditionalStartObject() throws IOException { + if (!isInsideOpenObject()) { + generator.writeStartObject(); + } else { + setInsideOpenObject(false); + } + } + + private void conditionalEndObject() { + if (isInsideOpenObject()) { + // This triggers if we were inside a data node with payload and + // subnodes, but none of the subnodes contained data + try { + generator.writeEndObject(); + setInsideOpenObject(false); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + } + } + + private void fieldName() { + if (hasFieldName) { + return; + } + + try { + generator.writeFieldName(TRACE); + } catch (IOException e) { + throw new TraceRenderWrapper(e); + } + hasFieldName = true; + } + + boolean isInsideOpenObject() { + return insideOpenObject; + } + + void setInsideOpenObject(boolean insideOpenObject) { + this.insideOpenObject = insideOpenObject; + } + public static final class TraceRenderWrapper extends RuntimeException { + + /** + * Should never be serialized, but this is still needed. + */ + private static final long serialVersionUID = 2L; + + TraceRenderWrapper(IOException wrapped) { + super(wrapped); + } + + } +} diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index 8bbb8500cfd..6c7878f99ed 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.yolean.trace.TraceNode; import org.junit.Test; +import java.io.IOException; import java.net.URI; import static org.junit.Assert.assertEquals; @@ -70,6 +72,36 @@ public class JSONLogTestCase { } @Test + public void test_json_of_trace() throws IOException { + TraceNode root = new TraceNode("root", 7); + AccessLogEntry entry = newAccessLogEntry("test"); + entry.setTrace(root); + + String expectedOutput = + "{\"ip\":\"152.200.54.243\"," + + "\"time\":920880005.023," + + "\"duration\":0.122," + + "\"responsesize\":9875," + + "\"code\":200," + + "\"method\":\"GET\"," + + "\"uri\":\"?query=test\"," + + "\"version\":\"HTTP/1.1\"," + + "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," + + "\"host\":\"localhost\"," + + "\"scheme\":null," + + "\"localport\":0," + + "\"trace\":{\"timestamp\":0,\"message\":\"root\"}," + + "\"search\":{" + + "\"totalhits\":1234," + + "\"hits\":0," + + "\"coverage\":{\"coverage\":100,\"documents\":100}" + + "}" + + "}"; + + assertEquals(expectedOutput, new JSONFormatter(entry).format()); + } + + @Test public void test_with_keyvalues() { AccessLogEntry entry = newAccessLogEntry("test"); entry.addKeyValue("singlevalue", "value1"); diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt index 341155457a8..6132c253c13 100644 --- a/container-core/CMakeLists.txt +++ b/container-core/CMakeLists.txt @@ -8,6 +8,7 @@ install_config_definition(src/main/resources/configdefinitions/identity.def cont install_config_definition(src/main/resources/configdefinitions/log-handler.def container.core.log-handler.def) install_config_definition(src/main/resources/configdefinitions/metrics-packets-handler.def container.jdisc.state.metrics-packets-handler.def) install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def) +install_config_definition(src/main/resources/configdefinitions/metrics-proxy-api.def container.handler.metrics.metrics-proxy-api.def) install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def) install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def) install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def) diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 91c0e4cc6bf..fe8fb91d71f 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -11,6 +11,23 @@ ], "fields": [] }, + "com.yahoo.container.handler.ClustersStatus$Require": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.container.handler.ClustersStatus$Require[] values()", + "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE", + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL" + ] + }, "com.yahoo.container.handler.ClustersStatus": { "superClass": "com.yahoo.component.AbstractComponent", "interfaces": [], @@ -23,7 +40,8 @@ "public void setReceiveTrafficByDefault(boolean)", "public void setUp(java.lang.Object)", "public void setDown(java.lang.Object)", - "public boolean containerShouldReceiveTraffic()" + "public boolean containerShouldReceiveTraffic()", + "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)" ], "fields": [] }, diff --git a/container-core/pom.xml b/container-core/pom.xml index f3861c92129..e51d99c3b78 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -16,6 +16,16 @@ <packaging>container-plugin</packaging> <dependencies> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>http-utils</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -227,6 +237,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-standalone</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index aab13a1cc7b..0ed0daa2141 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent { @Inject public ClustersStatus() { } + public enum Require {ONE, ALL} + /** Are there any (in-service influencing) clusters in this container? */ private boolean containerHasClusters; @@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent { setDown((String) clusterIdentifier); } - /** Returns whether this container should receive traffic based on the state of this */ + @Deprecated // TODO: Remove on Vespa 8 public boolean containerShouldReceiveTraffic() { + return containerShouldReceiveTraffic(Require.ONE); + } + /** + * Returns whether this container should receive traffic based on the state of this + * @param require Requirement for being up, ALL or ONE. + */ + public boolean containerShouldReceiveTraffic(Require require) { synchronized (mutex) { if (containerHasClusters) { - // Should receive traffic when at least one cluster is up - return clusterStatus.values().stream().anyMatch(status -> status==true); + switch (require) { + case ONE: + // Should receive traffic when at least one cluster is up + return clusterStatus.values().stream().anyMatch(status -> status == true); + case ALL: + default: + return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true); + } } else { return true; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index b9ef1627ce7..f712690efc5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -102,10 +102,15 @@ public class VipStatus { private void updateCurrentlyInRotation() { synchronized (mutex) { - if (rotationOverride != null) + if (rotationOverride != null) { currentlyInRotation = rotationOverride; - else - currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + } else { + if (healthState.status() == StateMonitor.Status.up) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE); + } else { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL); + } + } // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' if (currentlyInRotation) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java index 9bd30d287d4..321f7b3994a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http; +package com.yahoo.container.handler.metrics; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java index aa82a921e1a..92840cee48f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http; +package com.yahoo.container.handler.metrics; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java index 9de5933bd1f..def06ce9de3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http; +package com.yahoo.container.handler.metrics; import com.yahoo.container.jdisc.HttpResponse; diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java new file mode 100644 index 00000000000..78ea62e1b3a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java @@ -0,0 +1,77 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; +import com.yahoo.yolean.Exceptions; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; + +/** + * @author gjoranv + */ +public class MetricsV2Handler extends HttpHandlerBase { + + public static final String V2_PATH = "/metrics/v2"; + static final String VALUES_PATH = V2_PATH + "/values"; + + private static final int HTTP_CONNECT_TIMEOUT = 5000; + private static final int HTTP_SOCKET_TIMEOUT = 30000; + + private final String metricsProxyUri; + private final HttpClient httpClient = createHttpClient(); + + @Inject + public MetricsV2Handler(Executor executor, + MetricsProxyApiConfig config) { + super(executor); + metricsProxyUri = "http://localhost:" + config.metricsPort() + config.metricsApiPath(); + } + + @Override + protected Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + String uri = metricsProxyUri + consumerQuery(consumer); + String metricsJson = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); + return new JsonResponse(OK, metricsJson); + } catch (IOException e) { + log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static CloseableHttpClient createHttpClient() { + return VespaHttpClientBuilder.create() + .setUserAgent("application-metrics-retriever") + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout(HTTP_CONNECT_TIMEOUT) + .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .build()) + .build(); + } + + static String consumerQuery(String consumer) { + return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer; + } +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java new file mode 100644 index 00000000000..031886afc94 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** Exported config package */ +@ExportPackage +package com.yahoo.container.handler.metrics; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index dddde1205ca..a9ea737c96c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -93,7 +93,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler /** Render and return whether the channel was closed */ private void render(HttpRequest request, HttpResponse httpResponse, - LazyContentChannel channel, long startTime) throws IOException { + LazyContentChannel channel, long startTime) { LoggingCompletionHandler logOnCompletion = null; ContentChannelOutputStream output = null; try { @@ -168,7 +168,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public void close(CompletionHandler completionHandler) { if ( closed ) return; - try { httpRequest.getData().close(); } catch (IOException e) {}; + try { httpRequest.getData().close(); } catch (IOException e) {} if (channel == null) channel = handleResponse(); try { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index f690c240537..faa08402cdc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -14,6 +14,7 @@ import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** @@ -27,7 +28,7 @@ public class StateMonitor extends AbstractComponent { private final static Logger log = Logger.getLogger(StateMonitor.class.getName()); - public enum Status {up, down, initializing}; + public enum Status {up, down, initializing} private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); private final Thread thread; @@ -37,6 +38,7 @@ public class StateMonitor extends AbstractComponent { private volatile MetricSnapshot snapshot; private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); + private final AtomicBoolean stopped = new AtomicBoolean(false); /** For testing */ public StateMonitor() { @@ -53,10 +55,16 @@ public class StateMonitor extends AbstractComponent { } StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { + this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)), + Status.valueOf(config.initialStatus()), + timer, threadFactory); + } + /* For Testing */ + public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) { this.timer = timer; - this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); + this.snapshotIntervalMs = snapshotIntervalMS; this.lastSnapshotTimeMs = timer.currentTimeMillis(); - this.status = Status.valueOf(config.initialStatus()); + this.status = status; thread = threadFactory.newThread(this::run); thread.start(); } @@ -99,13 +107,13 @@ public class StateMonitor extends AbstractComponent { private void run() { log.finest("StateMonitor started."); try { - while (!Thread.interrupted()) { - checkTime(); - Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + synchronized (stopped) { + while (!stopped.get()) { + checkTime(); + stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + } } - } catch (InterruptedException e) { - - } + } catch (InterruptedException e) { } log.finest("StateMonitor stopped."); } @@ -137,12 +145,13 @@ public class StateMonitor extends AbstractComponent { @Override public void deconstruct() { - thread.interrupt(); + synchronized (stopped) { + stopped.set(true); + stopped.notifyAll(); + } try { thread.join(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + } catch (InterruptedException e) { } if (thread.isAlive()) { log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); } diff --git a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java index 17ed321331b..32ea3ae708f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java @@ -1,33 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; - /** * A 200 ok response with a message in JSON. * * @author bratseth */ -public class MessageResponse extends HttpResponse { - - private final Slime slime = new Slime(); +public class MessageResponse extends SlimeJsonResponse { public MessageResponse(String message) { - super(200); - slime.setObject().setString("message", message); + super(slime(message)); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime slime(String message) { + var slime = new Slime(); + slime.setObject().setString("message", message); + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index ff301d44798..0188136addb 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -1,46 +1,41 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; +import java.util.List; /** * Returns a response containing an array of links to sub-resources * * @author bratseth */ -public class ResourceResponse extends HttpResponse { +public class ResourceResponse extends SlimeJsonResponse { - private final Slime slime = new Slime(); + public ResourceResponse(URI parentUrl, List<String> subResources) { + super(200, toSlime(parentUrl, subResources)); + } public ResourceResponse(URI parentUrl, String ... subResources) { - super(200); - Cursor resourceArray = slime.setObject().setArray("resources"); - for (String subResource : subResources) { - Cursor resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(parentUrl).append(subResource) - .withTrailingSlash() - .toString()); - } + this(parentUrl, List.of(subResources)); } public ResourceResponse(HttpRequest request, String ... subResources) { this(request.getUri(), subResources); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime toSlime(URI parentUrl, List<String> subResources) { + var slime = new Slime(); + var resourceArray = slime.setObject().setArray("resources"); + for (var subResource : subResources) { + var resourceEntry = resourceArray.addObject(); + resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + .withTrailingSlash() + .toString()); + } + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-core/src/main/resources/configdefinitions/container-http.def b/container-core/src/main/resources/configdefinitions/container-http.def index ccf559b862a..23edd402893 100644 --- a/container-core/src/main/resources/configdefinitions/container-http.def +++ b/container-core/src/main/resources/configdefinitions/container-http.def @@ -3,3 +3,6 @@ namespace=container.core ## If non-empty, handlers should emit a header containing this string as key and the local host name as value hostResponseHeaderKey string default="" + +## For debugging, number of requests to add trace and timing information too if debugging is enabled. +numQueriesToTraceOnDebugAfterConstruction int default=1000 diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def index 5e70c72ae3f..4e91d85b2b8 100644 --- a/container-core/src/main/resources/configdefinitions/health-monitor.def +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -6,4 +6,4 @@ namespace=container.jdisc.config snapshot_interval double default=300 # Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values -initialStatus string default="up" +initialStatus string default="initializing" diff --git a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def index d80b48c9093..3e5b973e3f3 100644 --- a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt +++ b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def @@ -1,2 +1,6 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.4) + +namespace=container.handler.metrics + +metricsPort int +metricsApiPath string diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 52679c15957..d3479936544 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -4,6 +4,8 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.jdisc.state.StateMonitor; +import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; /** @@ -12,45 +14,79 @@ import org.junit.Test; * @author steinar */ public class VipStatusTestCase { + private static final String [] clusters = {"cluster1", "cluster2", "cluster3"}; - @Test - public void testVipStatusWorksWithClusters() { + private static QrSearchersConfig getSearchersCfg() { var b = new QrSearchersConfig.Builder(); var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name("cluster1"); - searchClusterB.name("cluster2"); - searchClusterB.name("cluster3"); + for (String cluster : clusters) { + searchClusterB.name(cluster); + } b.searchcluster(searchClusterB); - VipStatus v = new VipStatus(b.build()); + return b.build(); + } + private static VipStatus getVipStatus(StateMonitor.Status startState) { + return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> { + Thread thread = new Thread(runnable, "StateMonitor"); + thread.setDaemon(true); + return thread; + })); + } - String cluster1 = "cluster1"; - String cluster2 = "cluster2"; - String cluster3 = "cluster3"; + private static void removeAll(VipStatus v) { + for (String s : clusters) { + v.removeFromRotation(s); + } + } + private static void addAll(VipStatus v) { + for (String s : clusters) { + v.addToRotation(s); + } + } + private static void verifyUpOrDown(StateMonitor.Status status) { + VipStatus v = getVipStatus(status); + removeAll(v); // initial state assertFalse(v.isInRotation()); - - // one cluster becomes up - v.addToRotation(cluster1); + v.addToRotation(clusters[0]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[1]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[2]); assertTrue(v.isInRotation()); + } - // all clusters down - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + @Test + public void testInitializingOrDownRequireAllUp() { + verifyUpOrDown(StateMonitor.Status.initializing); + verifyUpOrDown(StateMonitor.Status.down); + } + + @Test + public void testUpRequireAllDown() { + VipStatus v = getVipStatus(StateMonitor.Status.initializing); assertFalse(v.isInRotation()); - // some clusters down - v.addToRotation(cluster2); + addAll(v); assertTrue(v.isInRotation()); - // all clusters up - v.addToRotation(cluster1); - v.addToRotation(cluster3); + + v.removeFromRotation(clusters[0]); + assertTrue(v.isInRotation()); + v.removeFromRotation(clusters[1]); assertTrue(v.isInRotation()); - // and down again - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + v.removeFromRotation(clusters[2]); + assertFalse(v.isInRotation()); // All down + v.addToRotation(clusters[1]); assertFalse(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); // All up + v.removeFromRotation(clusters[0]); + v.removeFromRotation(clusters[2]); + assertTrue(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/ErrorResponseTest.java index ccf04560906..882f9044dce 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/ErrorResponseTest.java @@ -1,5 +1,5 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.http; +package com.yahoo.container.handler.metrics; import org.junit.Test; diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java new file mode 100644 index 00000000000..b57814e50aa --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java @@ -0,0 +1,143 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.V2_PATH; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.VALUES_PATH; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public class MetricsV2HandlerTest { + + private static final String URI_BASE = "http://localhost"; + + private static final String V2_URI = URI_BASE + V2_PATH; + private static final String VALUES_URI = URI_BASE + VALUES_PATH; + + // Mock applicationmetrics api + private static final String MOCK_METRICS_PATH = "/node0"; + + private static final String TEST_FILE = "application-metrics.json"; + private static final String RESPONSE = getFileContents(TEST_FILE); + private static final String CPU_METRIC = "cpu.util"; + private static final String REPLACED_CPU_METRIC = "replaced_cpu_util"; + private static final String CUSTOM_CONSUMER = "custom-consumer"; + + private static RequestHandlerTestDriver testDriver; + + @Rule + public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort()); + + @Before + public void setup() { + setupWireMock(); + var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), + new MetricsProxyApiConfig.Builder() + .metricsPort(wireMockRule.port()) + .metricsApiPath(MOCK_METRICS_PATH) + .build()); + testDriver = new RequestHandlerTestDriver(handler); + } + + private void setupWireMock() { + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .willReturn(aResponse().withBody(RESPONSE))); + + // Add a slightly different response for a custom consumer. + String myConsumerResponse = RESPONSE.replaceAll(CPU_METRIC, REPLACED_CPU_METRIC); + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .withQueryParam("consumer", equalTo(CUSTOM_CONSUMER)) + .willReturn(aResponse().withBody(myConsumerResponse))); + } + + @Test + public void v2_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(V2_URI).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUri = resources.getJSONObject(0); + assertEquals(VALUES_URI, valuesUri.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() throws Exception { + JSONObject responseJson = getResponseAsJson(null); + System.out.println(responseJson.toString(4)); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(V2_URI + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + assertTrue(root.getString("error" ).startsWith("No content")); + } + + @Test + public void values_response_is_equal_to_test_file() { + String response = testDriver.sendRequest(VALUES_URI).readAll(); + assertEquals(RESPONSE, response); + } + + @Test + public void consumer_is_propagated_to_metrics_proxy_api() throws JSONException { + JSONObject responseJson = getResponseAsJson(CUSTOM_CONSUMER); + + JSONObject firstNodeMetricsValues = + responseJson.getJSONArray("nodes").getJSONObject(0) + .getJSONObject("node") + .getJSONArray("metrics").getJSONObject(0) + .getJSONObject("values"); + + assertTrue(firstNodeMetricsValues.has(REPLACED_CPU_METRIC)); + } + + private JSONObject getResponseAsJson(String consumer) { + String response = testDriver.sendRequest(VALUES_URI + consumerQuery(consumer)).readAll(); + try { + return new JSONObject(response); + } catch (JSONException e) { + fail("Failed to create json object: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private static String getFileContents(String filename) { + InputStream in = MetricsV2HandlerTest.class.getClassLoader().getResourceAsStream(filename); + if (in == null) { + throw new RuntimeException("File not found: " + filename); + } + return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); + } + +} diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index 78541137db5..8a1640e2c0e 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -57,7 +57,8 @@ public class StateHandlerTestBase { HealthMonitorConfig healthMonitorConfig = new HealthMonitorConfig( new HealthMonitorConfig.Builder() - .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL))); + .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)) + .initialStatus("up")); ThreadFactory threadFactory = ignored -> mock(Thread.class); this.monitor = new StateMonitor(healthMonitorConfig, timer, threadFactory); builder.guiceModules().install(new AbstractModule() { diff --git a/container-core/src/test/resources/application-metrics.json b/container-core/src/test/resources/application-metrics.json new file mode 100644 index 00000000000..52cbb721bb1 --- /dev/null +++ b/container-core/src/test/resources/application-metrics.json @@ -0,0 +1,92 @@ +{ + "nodes": [ + { + "hostname": "node0", + "role": "role0", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 16.222 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "queries.count": 4 + }, + "dimensions": { + "documentType": "music" + } + } + ] + } + ] + }, + { + "hostname": "node1", + "role": "role1", + "node": { + "timestamp": 1234, + "metrics": [ + { + "values": { + "cpu.util": 32.444 + }, + "dimensions": { + "state": "active" + } + } + ] + }, + "services": [ + { + "name": "searchnode", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "queries.count": 8 + }, + "dimensions": { + "documentType": "music" + } + } + ] + }, + { + "name": "slobrok", + "timestamp": 1234, + "status": { + "code": "unknown", + "description": "Unable to fetch metrics from service 'slobrok'" + }, + "metrics": [ + { + "values": {}, + "dimensions": { + "instance": "slobrok0" + } + } + ] + } + ] + } + ] +} diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 30cdbbfe531..a621545446f 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.25.v20191220</jetty.version> + <jetty.version>9.4.26.v20200117</jetty.version> <lz4.version>1.3.0</lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index a7217d05315..738a4cc8700 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -106,6 +106,10 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index 35280e12146..81de014c6ad 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -14,6 +14,8 @@ "public abstract javax.net.ssl.SSLContext getRoleSslContext(java.lang.String, java.lang.String)", "public abstract java.lang.String getRoleToken(java.lang.String)", "public abstract java.lang.String getRoleToken(java.lang.String, java.lang.String)", + "public abstract java.lang.String getAccessToken(java.lang.String)", + "public abstract java.lang.String getAccessToken(java.lang.String, java.util.List)", "public abstract java.util.List getIdentityCertificate()", "public abstract java.security.PrivateKey getPrivateKey()" ], @@ -31,6 +33,17 @@ ], "fields": [] }, + "com.yahoo.container.jdisc.secretstore.SecretNotFoundException": { + "superClass": "java.lang.RuntimeException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)" + ], + "fields": [] + }, "com.yahoo.container.jdisc.secretstore.SecretStore": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java index 9045db3eda5..bb862aeca82 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java @@ -59,6 +59,16 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr } @Override + public String getAccessToken(String domain) { + throw new UnsupportedOperationException(message); + } + + @Override + public String getAccessToken(String domain, List<String> roles) { + throw new UnsupportedOperationException(message); + } + + @Override public List<X509Certificate> getIdentityCertificate() { throw new UnsupportedOperationException(message); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index 1db3f4a3b42..696aab85b0c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -16,6 +16,8 @@ public interface AthenzIdentityProvider { SSLContext getRoleSslContext(String domain, String role); String getRoleToken(String domain); String getRoleToken(String domain, String role); + String getAccessToken(String domain); + String getAccessToken(String domain, List<String> roles); List<X509Certificate> getIdentityCertificate(); PrivateKey getPrivateKey(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index 371f29e86a3..3a153ec3d8a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -19,6 +19,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -50,11 +52,11 @@ public class Deconstructor implements ComponentDeconstructor { } } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarly to AbstractComponent - log.info("Starting deconstruction of provider " + component); + log.log(DEBUG, () -> "Starting deconstruction of provider " + component); ((Provider<?>) component).deconstruct(); - log.info("Finished deconstruction of provider " + component); + log.log(DEBUG, () -> "Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { - log.info("Releasing container reference to resource " + component); + log.log(DEBUG, () -> "Releasing container reference to resource " + component); // No need to delay release, as jdisc does ref-counting ((SharedResource) component).release(); } @@ -70,8 +72,7 @@ public class Deconstructor implements ComponentDeconstructor { private final Collection<AbstractComponent> components; private final Collection<Bundle> bundles; - DestructComponentTask(Collection<AbstractComponent> components, - Collection<Bundle> bundles) { + DestructComponentTask(Collection<AbstractComponent> components, Collection<Bundle> bundles) { this.components = components; this.bundles = bundles; } @@ -88,10 +89,10 @@ public class Deconstructor implements ComponentDeconstructor { @Override public void run() { for (var component : components) { - log.info("Starting deconstruction of component " + component); + log.log(DEBUG, () -> "Starting deconstruction of component " + component); try { component.deconstruct(); - log.info("Finished deconstructing of component " + component); + log.log(DEBUG, () -> "Finished deconstructing of component " + component); } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded log.log(WARNING, "Exception thrown when deconstructing component " + component, e); } catch (Error e) { @@ -111,7 +112,7 @@ public class Deconstructor implements ComponentDeconstructor { // It should now be safe to uninstall the old bundles. for (var bundle : bundles) { try { - log.info("Uninstalling bundle " + bundle); + log.log(INFO, "Uninstalling bundle " + bundle); bundle.uninstall(); } catch (BundleException e) { log.log(SEVERE, "Could not uninstall bundle " + bundle); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java new file mode 100644 index 00000000000..b9439432c06 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.secretstore; + +/** + * @author mortent + */ +public class SecretNotFoundException extends RuntimeException { + + public SecretNotFoundException(String message) { + super(message); + } +} diff --git a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java index 00272778a2b..7ca5d2a6b10 100644 --- a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java +++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java @@ -72,6 +72,7 @@ public class GUIHandlerTest { private String servicesXml() { return "<container version='1.0'>\n" + + " <accesslog type='disabled'/>\n" + " <handler id='com.yahoo.search.query.gui.GUIHandler'>\n" + " <binding>http://*/querybuilder/*</binding>\n" + " </handler>\n" + diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b5fbe235c43..82d3223c8fe 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1954,6 +1954,7 @@ "methods": [ "public void <init>(com.yahoo.search.cluster.NodeManager)", "public void <init>(com.yahoo.search.cluster.NodeManager, boolean)", + "public void start()", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public void add(java.lang.Object, boolean)", "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)", @@ -1978,8 +1979,8 @@ "methods": [ "public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)", "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)", - "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", - "public final void ping(java.lang.Object, java.util.concurrent.Executor)", + "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", + "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)", "protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)", "public final com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)", @@ -2074,7 +2075,8 @@ "methods": [ "public abstract void working(java.lang.Object)", "public abstract void failed(java.lang.Object)", - "public abstract void ping(java.lang.Object, java.util.concurrent.Executor)", + "public void ping(java.lang.Object, java.util.concurrent.Executor)", + "public void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "public void pingIterationCompleted()" ], "fields": [] @@ -5788,6 +5790,7 @@ "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", + "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)", "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public boolean isComplete(java.lang.StringBuilder, java.util.Map)", "public com.yahoo.search.query.profile.QueryProfileProperties clone()", @@ -6283,6 +6286,7 @@ "public boolean isOverridable(java.lang.String)", "public java.lang.Class getValueClass(java.lang.String)", "public com.yahoo.search.query.profile.types.QueryProfileType getType(java.lang.String)", + "public com.yahoo.search.query.profile.types.FieldType getFieldType(com.yahoo.processing.request.CompoundName)", "public com.yahoo.search.query.profile.types.FieldDescription getField(java.lang.String)", "public com.yahoo.search.query.profile.types.FieldDescription removeField(java.lang.String)", "public void addField(com.yahoo.search.query.profile.types.FieldDescription)", @@ -6935,7 +6939,8 @@ "com.yahoo.search.rendering.JsonRenderer$FieldConsumer": { "superClass": "java.lang.Object", "interfaces": [ - "com.yahoo.search.result.Hit$RawUtf8Consumer" + "com.yahoo.search.result.Hit$RawUtf8Consumer", + "com.yahoo.container.logging.TraceRenderer$FieldConsumer" ], "attributes": [ "public" @@ -6947,6 +6952,7 @@ "protected boolean shouldRender(java.lang.String, java.lang.Object)", "protected boolean shouldRenderUtf8Value(java.lang.String, int)", "protected void renderFieldContents(java.lang.Object)", + "public void accept(java.lang.Object)", "public bridge synthetic void accept(java.lang.Object, java.lang.Object)" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 65d5879b004..365ee299ca4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -26,6 +26,7 @@ import java.util.Set; public class Index { public static class Attribute { + private boolean tokenizedContent = false; public final String name; @@ -207,20 +208,12 @@ public class Index { } } - /** - * Whether terms in this field are lower cased when indexing. - * - * @param lowercase true if terms are lowercased - */ + /** Sets whether terms in this field are lowercased when indexing. */ public void setLowercase(boolean lowercase) { this.lowercase = lowercase; } - /** - * Whether terms in this field are lower cased when indexing. - * - * @return true if terms are lowercased - */ + /** Returns whether terms in this field are lowercased when indexing. */ public boolean isLowercase() { return lowercase; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java index bac227ac3e3..55e16804602 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java @@ -31,14 +31,17 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { } } + @Override public ItemType getItemType() { return ItemType.AND; } + @Override public String getName() { return "SAND"; } + @Override public String getIndexName() { if (getItemCount() == 0) { return ""; @@ -54,4 +57,5 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { i.next().setWeight(w); } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java index 13673144a0a..d0ffcd2d0e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java @@ -3,10 +3,9 @@ package com.yahoo.prelude.query; /** - * An interface used for anything which represents a single block - * of query input. + * An interface used for anything which represents a single block of query input. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface BlockItem extends HasIndexItem { @@ -39,4 +38,5 @@ public interface BlockItem extends HasIndexItem { * is necessary to change operator? */ SegmentingRule getSegmentingRule(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java index a06009e642a..300d40d4366 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java @@ -79,4 +79,5 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements super.disclose(discloser); discloser.addProperty("index", index); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java index 69131b6c690..56554e14d01 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java @@ -28,17 +28,15 @@ public class NearItem extends CompositeItem { /** * Creates a <i>near</i> item with a limit to the distance between the words. * - * @param distance the number of word position which may separate - * the words for this near item to match + * @param distance the maximum position difference between the words which should be counted as a match */ public NearItem(int distance) { setDistance(distance); } public void setDistance(int distance) { - if (distance < 0) { - throw new IllegalArgumentException("Can not use negative distance '" + distance + "'."); - } + if (distance < 0) + throw new IllegalArgumentException("Can not use negative distance " + distance); this.distance = distance; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index ee4c0d4d9f0..5e292a06b0f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -520,7 +520,7 @@ abstract class StructuredParser extends AbstractParser { /** Returns a word, a phrase, or another composite */ private Item phraseBody(String indexName) { boolean quoted = false; - PhraseItem phrase = null; + CompositeItem composite = null; Item firstWord = null; boolean starAfterFirst = false; boolean starBeforeFirst; @@ -539,7 +539,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord(indexName, (firstWord != null) || (phrase != null)); + Item word = phraseWord(indexName, (firstWord != null) || (composite != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { @@ -555,34 +555,37 @@ abstract class StructuredParser extends AbstractParser { ((PhraseSegmentItem) word).setExplicit(true); } - if (phrase != null) { - phrase.addItem(word); + if (composite != null) { + composite.addItem(word); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); if (submodes.site) uriItem.setEndAnchorDefault(true); - phrase = uriItem; + composite = uriItem; } else { - phrase = new PhraseItem(); + if (quoted || indexFacts.getIndex(indexName).getPhraseSegmenting()) + composite = new PhraseItem(); + else + composite = new AndItem(); } - if (quoted || submodes.site || submodes.url) { - phrase.setExplicit(true); + if ( (quoted || submodes.site || submodes.url) && composite instanceof PhraseItem) { + ((PhraseItem)composite).setExplicit(true); } if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); - phrase.addItem(word); + composite.addItem(firstWord); + composite.addItem(word); } else if (word instanceof PhraseItem) { - phrase = (PhraseItem) word; + composite = (PhraseItem)word; } else { firstWord = word; starAfterFirst = tokens.skipNoIgnore(STAR); @@ -609,29 +612,29 @@ abstract class StructuredParser extends AbstractParser { braceLevelURL = 0; - if (phrase != null) { + if (composite != null) { if (addEndMarking()) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - return phrase; + return composite; } else if (firstWord != null && submodes.site) { if (starAfterFirst && !addStartOfHostMarker) { return firstWord; } else { - phrase = new PhraseItem(); + composite = new PhraseItem(); + ((PhraseItem)composite).setExplicit(true); if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); + composite.addItem(firstWord); if (!starAfterFirst) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - phrase.setExplicit(true); - return phrase; + return composite; } } else { if (firstWord != null && firstWord instanceof TermItem && (starAfterFirst || starBeforeFirst)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index fdd6ad47a98..ce13045b518 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -111,25 +111,17 @@ public class NormalizingSearcher extends Searcher { } private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) { - if (!block.isNormalizable()) { - return; - } - { - Index index = indexFacts.getIndex(block.getIndexName()); - if (index.isAttribute()) { - return; - } - if (!index.getNormalize()) { - return; - } - } + if ( ! block.isNormalizable()) return; + + Index index = indexFacts.getIndex(block.getIndexName()); + if (index.isAttribute()) return; + if ( ! index.getNormalize()) return; List<Alternative> terms = block.getAlternatives(); for (Alternative term : terms) { String accentDropped = linguistics.getTransformer().accentDrop(term.word, language); - if (!term.word.equals(accentDropped) && accentDropped.length() > 0) { + if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0) block.addTerm(accentDropped, term.exactness * .7d); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java index 84c793a6df1..5a936d42ccc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java @@ -4,6 +4,8 @@ package com.yahoo.prelude.querytransform; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.HasIndexItem; +import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NotItem; @@ -169,7 +171,9 @@ public class QueryRewrite { removeOtherNonrankedChildren(item, i); recall = Recall.RECALLS_EVERYTHING; } else if ((item instanceof AndItem) || (item instanceof NearItem)) { - item.removeItem(i); + if ( ! isRanked(item.getItem(i))) { + item.removeItem(i); + } } else if (item instanceof RankItem) { // empty } else { @@ -200,6 +204,20 @@ public class QueryRewrite { parent.removeItem(i); } } + + private static boolean isRanked(Item item) { + if (item instanceof CompositeItem) { + for (Item child : ((CompositeItem)item).items()) + if (isRanked(child)) return true; + return false; + } + else if (item instanceof HasIndexItem && Hit.SDDOCNAME_FIELD.equals(((HasIndexItem)item).getIndexName())) { + return false; // No point in ranking by sddocname + } + else { + return item.isRanked(); + } + } private static Item collapseSingleComposites(Item item) { if (!(item instanceof CompositeItem)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 655fbf6acc3..9a9044def2d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -188,13 +188,10 @@ public class StemmingSearcher extends Searcher { return (Item) w; } - if (context.isCJK) { - composite = chooseCompositeForCJK(current, - ((Item) current).getParent(), - indexName); - } else { - composite = phraseSegment(current, indexName); - } + if (context.isCJK) + composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName); + else + composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); @@ -331,39 +328,34 @@ public class StemmingSearcher extends Searcher { } } + private CompositeItem chooseComposite(BlockItem current, CompositeItem parent, String indexName) { + if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) + return createPhraseSegment(current, indexName); + else + return createAndSegment(current); + + } + private CompositeItem chooseCompositeForCJK(BlockItem current, CompositeItem parent, String indexName) { - CompositeItem composite; - if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) { - if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) { - composite = phraseSegment(current, indexName); - } else - composite = createAndSegment(current); - } else { - switch (current.getSegmentingRule()) { - case PHRASE: - composite = phraseSegment(current, indexName); - break; - case BOOLEAN_AND: - composite = createAndSegment(current); - break; + if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) + return chooseComposite(current, parent, indexName); + + switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only + case PHRASE: return createPhraseSegment(current, indexName); + case BOOLEAN_AND: return createAndSegment(current); default: - throw new IllegalArgumentException( - "Unknown segmenting rule: " - + current.getSegmentingRule() - + ". This is a bug in Vespa, as the implementation has gotten out of sync." - + " Please create a ticket as soon as possible."); - } + throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + + ". This is a bug in Vespa, as the implementation has gotten out of sync." + + " Please create a ticket as soon as possible."); } - return composite; } private AndSegmentItem createAndSegment(BlockItem current) { return new AndSegmentItem(current.stringValue(), true, true); } - private CompositeItem phraseSegment(BlockItem current, String indexName) { - CompositeItem composite; - composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); + private CompositeItem createPhraseSegment(BlockItem current, String indexName) { + CompositeItem composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); composite.setIndexName(indexName); return composite; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index bbdb3b796a2..82148cf54e6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -119,6 +119,9 @@ public class ValidateSortingSearcher extends Searcher { String name = f.getFieldName(); if ("[rank]".equals(name) || "[docid]".equals(name)) { // built-in constants - ok + } else if ("[relevance]".equals(name)) { + // built-in constant '[relevance]' must map to '[rank]' + f.getSorter().setName("[rank]"); } else if ("[relevancy]".equals(name)) { // built-in constant '[relevancy]' must map to '[rank]' f.getSorter().setName("[rank]"); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 395d8853603..1e3f11f4f78 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -288,7 +288,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(""); } - /** * Construct a query from a string formatted in the http style, e.g <code>?query=test&offset=10&hits=13</code> * The query must be uri encoded. @@ -297,7 +296,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(query, null); } - /** * Creates a query from a request * diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java index dd01d895963..0d491d2f0c1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java @@ -79,6 +79,8 @@ public abstract class BaseNodeMonitor<T> { */ public abstract void responded(); + /** @deprecated Not used */ + @Deprecated // TODO: Remove on Vespa 8 public boolean isIdle() { return (now()-respondedAt) >= configuration.getIdleLimit(); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index d4b6279be89..15cf4995b77 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -38,6 +38,9 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); + /** @deprecated It is not advised to start the monitoring thread in the constructor. + * Use ClusterMonitor(NodeManager manager, false) and explicit start(). */ + @Deprecated public ClusterMonitor(NodeManager<T> manager) { this(manager, true); } @@ -50,6 +53,12 @@ public class ClusterMonitor<T> { } } + public void start() { + if ( ! monitorThread.isAlive()) { + monitorThread.start(); + } + } + /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } @@ -92,7 +101,7 @@ public class ClusterMonitor<T> { Boolean wasWorking = monitor.isKnownWorking(); monitor.responded(); if (wasWorking != monitor.isKnownWorking()) - nodeManager.working(monitor.getNode()); + nodeManager.working(node); } /** @@ -101,7 +110,7 @@ public class ClusterMonitor<T> { public void ping(Executor executor) { for (Iterator<BaseNodeMonitor<T>> i = nodeMonitorIterator(); i.hasNext() && !closed.get(); ) { BaseNodeMonitor<T> monitor= i.next(); - nodeManager.ping(monitor.getNode(), executor); // Cause call to failed or responded + nodeManager.ping(this, monitor.getNode(), executor); // Cause call to failed or responded } if (closed.get()) return; // Do nothing to change state if close has started. nodeManager.pingIterationCompleted(); diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 20f56c86f7b..2d05168731a 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -58,7 +58,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod this(id, connections, hasher, internal, true); } - public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { + protected ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { super(id); this.hasher = hasher; this.monitor = new ClusterMonitor<>(this, startPingThread); @@ -70,7 +70,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override - public final void ping(T p, Executor executor) { + public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { log(LogLevel.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask<Pong> future = new FutureTask<>(pinger); @@ -80,7 +80,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod Throwable logThrowable = null; try { - pong = future.get(monitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); + pong = future.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { pong = new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + p)); logThrowable = e; @@ -96,10 +96,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod future.cancel(true); if (pong.badResponse()) { - monitor.failed(p, pong.error().get()); + clusterMonitor.failed(p, pong.error().get()); log(LogLevel.FINE, "Failed ping - ", pong); } else { - monitor.responded(p); + clusterMonitor.responded(p); log(LogLevel.FINE, "Answered ping - ", p); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index 226e0180d2e..21e5fe3bc7f 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -15,22 +15,10 @@ public class MonitorConfiguration { private long checkInterval=1000; /** - * The number of times a failed node must respond before getting - * traffic again - */ - private int responseAfterFailLimit=3; - - /** - * The number of ms a node is allowed to stay idle before it is - * pinged - */ - private long idleLimit=3000; - - /** * The number of milliseconds to attempt to complete a request * before giving up */ - private long requestTimeout = 5000; + private final long requestTimeout = 980; /** * The number of milliseconds a node is allowed to fail before we @@ -39,17 +27,6 @@ public class MonitorConfiguration { private long failLimit=5000; /** - * The number of times a node is allowed to fail in one hour - * before it is quarantined for an hour - */ - private int failQuarantineLimit=3; - - /** - * The number of ms to quarantine an unstable node - */ - private long quarantineTime=1000*60*60; - - /** * Sets the interval between each ping of idle or failing nodes * Default is 1000ms */ @@ -68,25 +45,27 @@ public class MonitorConfiguration { /** * Sets the number of times a failed node must respond before it is put * back in service. Default is 3. + * @deprecated Will go away in Vespa 8 */ - public void setResponseAfterFailLimit(int responseAfterFailLimit) { - this.responseAfterFailLimit=responseAfterFailLimit; - } + @Deprecated + public void setResponseAfterFailLimit(int responseAfterFailLimit) { } /** * Sets the number of ms a node (failing or working) is allowed to * stay idle before it is pinged. Default is 3000 + * @deprecated Will go away in Vespa 8 */ - public void setIdleLimit(int idleLimit) { - this.idleLimit=idleLimit; - } + @Deprecated + public void setIdleLimit(int idleLimit) { } /** * Gets the number of ms a node (failing or working) * is allowed to stay idle before it is pinged. Default is 3000 + * @deprecated Will go away in Vespa 8 */ + @Deprecated public long getIdleLimit() { - return idleLimit; + return 3000; } /** @@ -112,28 +91,24 @@ public class MonitorConfiguration { * in quarantine. Once in quarantine it won't be put back in * productuion before quarantineTime has expired even if it is * working. Default is 3 + * @deprecated Will go away in Vespa 8 */ - public void setFailQuarantineLimit(int failQuarantineLimit) { - this.failQuarantineLimit=failQuarantineLimit; - } + @Deprecated + public void setFailQuarantineLimit(int failQuarantineLimit) { } /** * The number of ms an unstable node is quarantined. Default is * 100*60*60 + * @deprecated Will go away in Vespa 8 */ - public void setQuarantineTime(long quarantineTime) { - this.quarantineTime=quarantineTime; - } + @Deprecated + public void setQuarantineTime(long quarantineTime) { } public String toString() { return "monitor configuration [" + "checkInterval: " + checkInterval + - " responseAfterFailLimit: " + responseAfterFailLimit + - " idleLimit: " + idleLimit + " requestTimeout " + requestTimeout + - " feilLimit " + failLimit + - " failQuerantineLimit " + failQuarantineLimit + - " quarantineTime " + quarantineTime + + " failLimit " + failLimit + "]"; } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 9b20139e3c5..481f1e1b5a5 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -19,9 +19,21 @@ public interface NodeManager<T> { /** * Called when a node should be pinged. - * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. */ - void ping(T node, Executor executor); + @Deprecated + default void ping(T node, Executor executor) { + throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); + } + + /** + * Called when a node should be pinged. + * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded + */ + default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) { + ping(node, executor); + } /** Called right after a ping has been issued to each node. This default implementation does nothing. */ default void pingIterationCompleted() {} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 03b51fbaf70..423de07f8c4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -11,12 +11,13 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; @@ -58,6 +59,7 @@ public class Dispatcher extends AbstractComponent { /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; + private final ClusterMonitor clusterMonitor; private final LoadBalancer loadBalancer; @@ -82,49 +84,53 @@ public class Dispatcher extends AbstractComponent { public static QueryProfileType getArgumentType() { return argumentType; } @Inject - public Dispatcher(ComponentId clusterId, + public Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus), - dispatchConfig, - metric); - } + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(), + vipStatus, new RpcPingFactory(resourcePool)), + dispatchConfig, metric); - private Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { - this(searchCluster, - dispatchConfig, - new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster), - metric); } - /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, - DispatchConfig dispatchConfig, - RpcInvokerFactory rcpInvokerFactory, - Metric metric) { - this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric); + private Dispatcher(RpcResourcePool resourcePool, SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { + this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric); } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, + protected Dispatcher(ClusterMonitor clusterMonitor, + SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, - PingFactory pingFactory, Metric metric) { if (dispatchConfig.useMultilevelDispatch()) throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported"); this.searchCluster = searchCluster; + this.clusterMonitor = clusterMonitor; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); - - searchCluster.startClusterMonitoring(pingFactory); + searchCluster.addMonitoring(clusterMonitor); + try { + while ( ! searchCluster.hasInformationAboutAllNodes()) { + Thread.sleep(1); + } + } catch (InterruptedException e) {} + + /* + * No we have information from all nodes and a ping iteration has completed. + * Instead of waiting until next ping interval to update coverage and group state, + * we should compute the state ourselves, so that when the dispatcher is ready the state + * of its groups are also known. + */ + searchCluster.pingIterationCompleted(); } /** Returns the search cluster this dispatches to */ @@ -134,8 +140,8 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The seach cluster must be shutdown first as it uses the invokerfactory. */ - searchCluster.shutDown(); + /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + clusterMonitor.shutdown(); invokerFactory.release(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index 05ce6d50493..a2821892358 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -22,6 +22,7 @@ import java.util.List; * @author bratseth */ class RpcClient implements Client { + private final Supervisor supervisor; public RpcClient(int transportThreads) { @@ -66,7 +67,7 @@ class RpcClient implements Client { @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, - ResponseReceiver responseReceiver, double timeoutSeconds) { + ResponseReceiver responseReceiver, double timeoutSeconds) { Request request = new Request(rpcMethod); request.parameters().add(new Int8Value(compression.getCode())); request.parameters().add(new Int32Value(uncompressedLength)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 5c9928de924..74bc9e8bfbb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,28 +1,24 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; -import java.util.concurrent.Callable; /** * @author ollivir */ -public class RpcInvokerFactory extends InvokerFactory implements PingFactory { +public class RpcInvokerFactory extends InvokerFactory { /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); @@ -60,12 +56,4 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { return new RpcFillInvoker(rpcResourcePool, documentDb); } - public void release() { - rpcResourcePool.release(); - } - - @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - return new RpcPing(node, monitor, rpcResourcePool); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index e0f1dc5e675..5e04f1d7a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -10,55 +10,63 @@ import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.rpc.Client.ResponseOrError; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.result.ErrorMessage; import com.yahoo.yolean.Exceptions; -import java.util.concurrent.Callable; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; -public class RpcPing implements Callable<Pong> { +public class RpcPing implements Pinger, Client.ResponseReceiver { + private static final Logger log = Logger.getLogger(RpcPing.class.getName()); private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; private final Node node; private final RpcResourcePool resourcePool; private final ClusterMonitor<Node> clusterMonitor; + private final long pingSequenceId; + private final PongHandler pongHandler; - public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) { + public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool, PongHandler pongHandler) { this.node = node; this.resourcePool = rpcResourcePool; this.clusterMonitor = clusterMonitor; + pingSequenceId = node.createPingSequenceId(); + this.pongHandler = pongHandler; } @Override - public Pong call() throws Exception { + public void ping() { try { - var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1); - - sendPing(queue); + sendPing(); + } catch (RuntimeException e) { + pongHandler.handle(new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + + ": " + Exceptions.toMessageString(e)))); + } + } - var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS); - if (responseOrError == null) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); - } else if (responseOrError.error().isPresent()) { - return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); - } + private Pong toPong(ResponseOrError<ProtobufResponse> responseOrError) { + if (responseOrError == null) { + return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); + } else if (responseOrError.error().isPresent()) { + return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); + } + try { return decodeReply(responseOrError.response().get()); - } catch (RuntimeException e) { - return new Pong( - ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e))); + } catch (InvalidProtocolBufferException e) { + return new Pong(ErrorMessage.createBackendCommunicationError(e.getMessage())); } } - private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) { + private void sendPing() { var connection = resourcePool.getConnection(node.key()); var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -76,4 +84,14 @@ public class RpcPing implements Callable<Pong> { } } + @Override + public void receive(ResponseOrError<ProtobufResponse> response) { + if (node.isLastReceivedPong(pingSequenceId)) { + pongHandler.handle(toPong(response)); + } else { + //TODO Reduce to debug or remove once we have enumerated what happens here. + log.info("Pong " + pingSequenceId + " from node " + node.key() + " in group " + node.group() + + " with hostname " + node.hostname() + " received too late, latest is " + node.getLastReceivedPongId()); + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java new file mode 100644 index 00000000000..ac8f0a59c20 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -0,0 +1,18 @@ +package com.yahoo.search.dispatch.rpc; + +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; + +public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { + this.rpcResourcePool = rpcResourcePool; + } + @Override + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { + return new RpcPing(node, monitor, rpcResourcePool, pongHandler); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index ca2a0c9bfb0..065489ef9a0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,6 +2,9 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -23,7 +26,7 @@ import java.util.Random; * * @author ollivir */ -public class RpcResourcePool { +public class RpcResourcePool extends AbstractComponent { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -33,13 +36,15 @@ public class RpcResourcePool { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools; - public RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { + RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); } + @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { + super(); var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key @@ -73,7 +78,9 @@ public class RpcResourcePool { } } - public void release() { + @Override + public void deconstruct() { + super.deconstruct(); nodeConnectionPools.values().forEach(NodeConnectionPool::release); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 07d8439ff46..76240e55c98 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; -import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 2f70c37cd48..e93b633f09d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -21,6 +21,8 @@ public class Node { private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicLong pingSequence = new AtomicLong(0); + private final AtomicLong lastPong = new AtomicLong(0); public Node(int key, String hostname, int group) { this.key = key; @@ -28,6 +30,18 @@ public class Node { this.group = group; } + /** Give a monotonically increasing sequence number.*/ + public long createPingSequenceId() { return pingSequence.incrementAndGet(); } + /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/ + public boolean isLastReceivedPong(long pingId ) { + long last = lastPong.get(); + while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) { + last = lastPong.get(); + } + return last < pingId; + } + public long getLastReceivedPongId() { return lastPong.get(); } + /** Returns the unique and stable distribution key of this node */ public int key() { return key; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index b16fa941f68..2e07d8d61e6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -1,13 +1,11 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; -import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; -import java.util.concurrent.Callable; public interface PingFactory { - Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); + Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java new file mode 100644 index 00000000000..b4a7ccbf98c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -0,0 +1,12 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +/** + * Send a ping and ensure that the pong is propagated to the ponghandler. + * Should not wait as this should be done in parallel on all nodes. + * + * @author baldersheim + */ +public interface Pinger { + void ping(); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java new file mode 100644 index 00000000000..c0579b5d36e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -0,0 +1,12 @@ +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.prelude.Pong; + +/** + * Handle the Pong result of a Ping. + * + * @author baldersheim + */ +public interface PongHandler { + void handle(Pong pong); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 5f211c37917..7619cb34b77 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -10,7 +10,6 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; -import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.LinkedHashMap; @@ -18,13 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -43,9 +36,8 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; private final ImmutableList<Group> orderedGroups; - private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; - private PingFactory pingFactory; + private final PingFactory pingFactory; private long nextLogTime = 0; /** @@ -58,10 +50,12 @@ public class SearchCluster implements NodeManager<Node> { */ private final Optional<Node> localCorpusDispatchTarget; - public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, + VipStatus vipStatus, PingFactory pingFactory) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; this.vipStatus = vipStatus; + this.pingFactory = pingFactory; List<Node> nodes = toNodes(dispatchConfig); this.size = nodes.size(); @@ -84,29 +78,25 @@ public class SearchCluster implements NodeManager<Node> { this.nodesByHost = nodesByHostBuilder.build(); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), - size, - containerClusterSize, - nodesByHost, - groups); - - this.clusterMonitor = new ClusterMonitor<>(this); + size, + containerClusterSize, + nodesByHost, + groups); } - public void shutDown() { - clusterMonitor.shutdown(); + /* Testing only */ + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, + VipStatus vipStatus, PingFactory pingFactory) { + this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); } - public void startClusterMonitoring(PingFactory pingFactory) { - this.pingFactory = pingFactory; - + public void addMonitoring(ClusterMonitor clusterMonitor) { for (var group : orderedGroups) { for (var node : group.nodes()) clusterMonitor.add(node, true); } } - ClusterMonitor<Node> clusterMonitor() { return clusterMonitor; } - private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname, int searchClusterSize, int containerClusterSize, @@ -247,7 +237,7 @@ public class SearchCluster implements NodeManager<Node> { vipStatus.removeFromRotation(clusterId); } - private boolean hasInformationAboutAllNodes() { + public boolean hasInformationAboutAllNodes() { return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); } @@ -263,26 +253,33 @@ public class SearchCluster implements NodeManager<Node> { return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id(); } - /** Used by the cluster monitor to manage node status */ - @Override - public void ping(Node node, Executor executor) { - if (pingFactory == null) return; // not initialized yet - - FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor)); - executor.execute(futurePong); - Pong pong = getPong(futurePong, node); - futurePong.cancel(true); - - if (pong.badResponse()) { - clusterMonitor.failed(node, pong.error().get()); - } else { - if (pong.activeDocuments().isPresent()) { - node.setActiveDocuments(pong.activeDocuments().get()); + private static class PongCallback implements PongHandler { + private final ClusterMonitor<Node> clusterMonitor; + private final Node node; + PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { + this.node = node; + this.clusterMonitor = clusterMonitor; + } + @Override + public void handle(Pong pong) { + if (pong.badResponse()) { + clusterMonitor.failed(node, pong.error().get()); + } else { + if (pong.activeDocuments().isPresent()) { + node.setActiveDocuments(pong.activeDocuments().get()); + } + clusterMonitor.responded(node); } - clusterMonitor.responded(node); } } + /** Used by the cluster monitor to manage node status */ + @Override + public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) { + Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor)); + pinger.ping(); + } + private void pingIterationCompletedSingleGroup() { Group group = groups.values().iterator().next(); group.aggregateActiveDocuments(); @@ -353,20 +350,6 @@ public class SearchCluster implements NodeManager<Node> { return workingNodes + nodesAllowedDown >= nodesInGroup; } - private Pong getPong(FutureTask<Pong> futurePong, Node node) { - try { - return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + node)); - } catch (ExecutionException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + node)); - } catch (TimeoutException e) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out")); - } - } - /** * Calculate whether a subset of nodes in a group has enough coverage */ @@ -402,6 +385,7 @@ public class SearchCluster implements NodeManager<Node> { } private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { + if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index d6918675e87..60c5d42c531 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher { /** The name of the query property containing the source name added to the query to each source by this */ public final static CompoundName SOURCENAME = new CompoundName("sourceName"); public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); - /** Logging field name constants */ public static final String LOG_COUNT_PREFIX = "count_"; @@ -110,7 +109,7 @@ public class FederationSearcher extends ForkingSearcher { // for testing public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, false, PropagateSourceProperties.ALL, null); + this(searchChainResolver, false, PropagateSourceProperties.EVERY, null); } private FederationSearcher(SearchChainResolver searchChainResolver, @@ -271,13 +270,14 @@ public class FederationSearcher extends ForkingSearcher { outgoing.setTimeout(timeout); switch (propagateSourceProperties) { - case ALL: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, - Query.nativeProperties); + case EVERY: + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, null); + break; + case NATIVE: case ALL: + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, Query.nativeProperties); break; case OFFSET_HITS: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, - queryAndHits); + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, queryAndHits); break; } @@ -290,10 +290,21 @@ public class FederationSearcher extends ForkingSearcher { private void propagatePerSourceQueryProperties(Query original, Query outgoing, Window window, String sourceName, String providerName, List<CompoundName> queryProperties) { - for (CompoundName key : queryProperties) { - Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); - if (value != null) - outgoing.properties().set(key, value); + if (queryProperties == null) { + outgoing.setHits(window.hits); + outgoing.setOffset(window.offset); + original.properties().listProperties(CompoundName.fromComponents("provider", providerName)).forEach((k, v) -> + outgoing.properties().set(k, v)); + original.properties().listProperties(CompoundName.fromComponents("source", sourceName)).forEach((k, v) -> + outgoing.properties().set(k, v)); + } + else { + for (CompoundName key : queryProperties) { + Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); + if (value != null) + outgoing.properties().set(key, value); + if (value != null) System.out.println("Setting " + key + " = " + value); + } } } @@ -321,7 +332,7 @@ public class FederationSearcher extends ForkingSearcher { private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) + - " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; + " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; return ErrorMessage.createInvalidQueryParameter(message); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index 3602d21f7d8..d636d3bc925 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -23,6 +23,7 @@ import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.yolean.trace.TraceNode; /** * Wrap the result of a query as an HTTP response. @@ -36,8 +37,13 @@ public class HttpSearchResponse extends ExtendedResponse { private final Renderer<Result> rendererCopy; private final Timing timing; private final HitCounts hitCounts; + private final TraceNode trace; public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) { + this(status, result, query, renderer, null); + } + + HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) { super(status); this.query = query; this.result = result; @@ -45,6 +51,7 @@ public class HttpSearchResponse extends ExtendedResponse { this.timing = SearchResponse.createTiming(query, result); this.hitCounts = SearchResponse.createHitCounts(query, result); + this.trace = trace; populateHeaders(headers(), result.getHeaders(false)); } @@ -107,6 +114,9 @@ public class HttpSearchResponse extends ExtendedResponse { @Override public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) { super.populateAccessLogEntry(accessLogEntry); + if (trace != null) { + accessLogEntry.setTrace(trace); + } populateAccessLogEntry(accessLogEntry, getHitCounts()); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 0981c6e8dad..dad106570ab 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -29,7 +29,7 @@ import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -58,6 +58,7 @@ import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -98,6 +99,8 @@ public class SearchHandler extends LoggingRequestHandler { private final ExecutionFactory executionFactory; + private final AtomicLong numRequestsLeftToTrace; + private final class MeanConnections implements Callback { @Override @@ -125,6 +128,7 @@ public class SearchHandler extends LoggingRequestHandler { accessLog, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } @@ -136,6 +140,17 @@ public class SearchHandler extends LoggingRequestHandler { CompiledQueryProfileRegistry queryProfileRegistry, ExecutionFactory executionFactory, Optional<String> hostResponseHeaderKey) { + this(statistics, metric, executor, accessLog, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey); + } + + private SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ExecutionFactory executionFactory, + long numQueriesToTraceOnDebugAfterStartup, + Optional<String> hostResponseHeaderKey) { super(executor, accessLog, metric, true); log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; @@ -150,6 +165,7 @@ public class SearchHandler extends LoggingRequestHandler { .setCallback(new MeanConnections())); this.hostResponseHeaderKey = hostResponseHeaderKey; + this.numRequestsLeftToTrace = new AtomicLong(numQueriesToTraceOnDebugAfterStartup); } /** @deprecated use the other constructor */ @@ -215,7 +231,6 @@ public class SearchHandler extends LoggingRequestHandler { } - @SuppressWarnings("unchecked") private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) { Query query = new Query(); Result result = new Result(query, errorMessage); @@ -281,7 +296,10 @@ public class SearchHandler extends LoggingRequestHandler { // Transform result to response Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), - result, query, renderer); + result, query, renderer, + log.isLoggable(Level.FINE) + ? query.getContext(false).getTrace().traceNode() + : null); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); @@ -330,7 +348,13 @@ public class SearchHandler extends LoggingRequestHandler { Execution execution = executionFactory.newExecution(searchChain); query.getModel().setExecution(execution); - execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false)); + if (log.isLoggable(Level.FINE) && (numRequestsLeftToTrace.getAndDecrement() > 0)) { + query.setTraceLevel(Math.max(1, query.getTraceLevel())); + execution.trace().setForceTimestamps(true); + + } else { + execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false)); + } if (query.properties().getBoolean(DETAILED_TIMING_LOGGING, false)) { // check and set (instead of set directly) to avoid overwriting stuff from prepareForBreakdownAnalysis() execution.context().setDetailedDiagnostics(true); @@ -389,7 +413,7 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(LogLevel.DEBUG, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { if ("Comparison method violates its general contract!".equals(e.getMessage())) { @@ -401,7 +425,7 @@ public class SearchHandler extends LoggingRequestHandler { else { ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(LogLevel.DEBUG, error::getDetailedMessage); return new Result(query, error); } } catch (LinkageError | StackOverflowError e) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 7444c94f491..830a3f4ef81 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -47,7 +47,7 @@ public class Ranking implements Cloneable { public static final String PROPERTIES = "properties"; static { - argumentType =new QueryProfileType(RANKING); + argumentType = new QueryProfileType(RANKING); argumentType.setStrict(true); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(LOCATION, "string", "location")); @@ -63,7 +63,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature")); argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); - argumentTypeName=new CompoundName(argumentType.getId().getName()); + argumentTypeName = new CompoundName(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index c654edda6f5..775dca7c444 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -45,7 +45,7 @@ import com.yahoo.search.yql.VespaGroupingStep; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -159,7 +159,7 @@ public class SelectParser implements Parser { return new QueryTree(root); } - private Item walkJson(Inspector inspector){ + private Item walkJson(Inspector inspector) { Item[] item = {null}; inspector.traverse((ObjectTraverser) (key, value) -> { String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; @@ -197,8 +197,8 @@ public class SelectParser implements Parser { public List<VespaGroupingStep> getGroupingSteps(String grouping){ List<VespaGroupingStep> groupingSteps = new ArrayList<>(); - List<String> groupingOperations = getOperations(grouping); - for (String groupingString : groupingOperations){ + List<String> groupingOperations = toGroupingRequests(grouping); + for (String groupingString : groupingOperations) { GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); groupingSteps.add(groupingStep); @@ -206,24 +206,51 @@ public class SelectParser implements Parser { return groupingSteps; } - private List<String> getOperations(String grouping) { - List<String> operations = new ArrayList<>(); - Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get(); - if (inspector.field("error_message").valid()){ + /** Translates a list of grouping requests on JSON form to a list in the grouping language form */ + private List<String> toGroupingRequests(String groupingJson) { + Inspector inspector = SlimeUtils.jsonToSlime(groupingJson.getBytes()).get(); + if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } - inspector.traverse( (ArrayTraverser) (key, value) -> { - String groupingString = value.toString(); - groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")"); - groupingString = groupingString.substring(1, groupingString.length()); - operations.add(groupingString); - }); - + List<String> operations = new ArrayList<>(); + inspector.traverse((ArrayTraverser) (__, item) -> operations.add(toGroupingRequest(item))); return operations; } + private String toGroupingRequest(Inspector groupingJson) { + StringBuilder b = new StringBuilder(); + toGroupingRequest(groupingJson, b); + return b.toString(); + } + + private void toGroupingRequest(Inspector groupingJson, StringBuilder b) { + switch (groupingJson.type()) { + case ARRAY: + groupingJson.traverse((ArrayTraverser) (index, item) -> { + toGroupingRequest(item, b); + if (index + 1 < groupingJson.entries()) + b.append(","); + }); + break; + case OBJECT: + groupingJson.traverse((ObjectTraverser) (name, object) -> { + b.append(name); + b.append("("); + toGroupingRequest(object, b); + b.append(") "); + }); + break; + case STRING: + b.append(groupingJson.asString()); + break; + default: + b.append(groupingJson.toString()); + break; + } + } + private Item buildFunctionCall(String key, Inspector value) { switch (key) { case WAND: @@ -250,7 +277,7 @@ public class SelectParser implements Parser { }); } else if (inspector.type() == OBJECT){ - if (inspector.field("children").valid()){ + if (inspector.field("children").valid()) { inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { item.addItem(walkJson(new_value)); }); @@ -259,22 +286,22 @@ public class SelectParser implements Parser { } } - private Inspector getChildren(Inspector inspector){ + private Inspector getChildren(Inspector inspector) { if (inspector.type() == ARRAY){ return inspector; } else if (inspector.type() == OBJECT){ - if (inspector.field("children").valid()){ + if (inspector.field("children").valid()) { return inspector.field("children"); } - if (inspector.field(1).valid()){ + if (inspector.field(1).valid()) { return inspector.field(1); } } return null; } - private HashMap<Integer, Inspector> childMap(Inspector inspector){ + private HashMap<Integer, Inspector> childMap(Inspector inspector) { HashMap<Integer, Inspector> children = new HashMap<>(); if (inspector.type() == ARRAY){ inspector.traverse((ArrayTraverser) (index, new_value) -> { @@ -291,14 +318,14 @@ public class SelectParser implements Parser { return children; } - private Inspector getAnnotations(Inspector inspector){ + private Inspector getAnnotations(Inspector inspector) { if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ return inspector.field("attributes"); } return null; } - private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){ + private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation) { HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ annotation.traverse((ObjectTraverser) (index, new_value) -> { @@ -308,7 +335,7 @@ public class SelectParser implements Parser { return attributes; } - private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){ + private HashMap<String, Inspector> getAnnotationMap(Inspector inspector) { HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java index 99f1e26b221..11864e60cec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -135,8 +135,8 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple @Override public List<String> getDimensions() { - List<String> dimensions=super.getDimensions(); - if (dimensions!=null) return dimensions; + List<String> dimensions = super.getDimensions(); + if (dimensions != null) return dimensions; return backingProfile.getDimensions(); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java index 4dc9ade62e5..e32d2dc226d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java @@ -22,8 +22,7 @@ public class ModelObjectMap extends PropertyMap { */ @Override protected boolean shouldSet(CompoundName name, Object value) { - if (value == null) return true; - return ! FieldType.isLegalFieldValue(value); + return value != null && ! FieldType.isLegalFieldValue(value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index e9ccdd22f98..7ae18f96d86 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -100,7 +100,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable public QueryProfileType getType() { return type; } /** Sets the type of this, or set to null to not use any type checking in this profile */ - public void setType(QueryProfileType type) { this.type=type; } + public void setType(QueryProfileType type) { this.type = type; } /** Returns the virtual variants of this, or null if none */ public QueryProfileVariants getVariants() { return variants; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index ea79b10d779..701ea7690f4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.ArrayList; @@ -31,7 +32,12 @@ public class QueryProfileProperties extends Properties { /** Values which has been overridden at runtime, or null if none */ private Map<CompoundName, Object> values = null; - /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */ + + /** + * Query profile references which has been overridden at runtime, possibly to the null value to clear values, + * or null if none (i.e this is lazy). + * Earlier values has precedence + */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; /** Creates an instance from a profile, throws an exception if the given profile is null */ @@ -48,20 +54,21 @@ public class QueryProfileProperties extends Properties { public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { name = unalias(name, context); - Object value = null; - if (values != null) - value = values.get(name); - if (value == null) { - Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); - if (reference != null) - return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null + if (values != null && values.containsKey(name)) + return values.get(name); // Returns this value, even if null + + Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); + if (reference != null) { + if (reference.getSecond() == null) + return null; // cleared + else + return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // even if null } - if (value == null) - value = profile.get(name, context, substitution); - if (value == null) - value = super.get(name, context, substitution); - return value; + Object value = profile.get(name, context, substitution); + if (value != null) + return value; + return super.get(name, context, substitution); } /** @@ -70,7 +77,7 @@ public class QueryProfileProperties extends Properties { * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile */ @Override - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { // TODO: Refactor try { name = unalias(name, context); @@ -87,8 +94,10 @@ public class QueryProfileProperties extends Properties { // Check types if ( ! profile.getTypes().isEmpty()) { - for (int i = 0; i<name.size(); i++) { - QueryProfileType type = profile.getType(name.first(i), context); + QueryProfileType type = null; + for (int i = 0; i < name.size(); i++) { + if (type == null) // We're on the first iteration, or no type is explicitly specified + type = profile.getType(name.first(i), context); if (type == null) continue; String localName = name.get(i); FieldDescription fieldDescription = type.getField(localName); @@ -97,12 +106,19 @@ public class QueryProfileProperties extends Properties { // TODO: In addition to strictness, check legality along the way - if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); - if (value == null) - throw new IllegalArgumentException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); + if (fieldDescription != null) { + if (i == name.size() - 1) { // at the end of the path, check the assignment type + value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + if (value == null) + throw new IllegalArgumentException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); + } + else if (fieldDescription.getType() instanceof QueryProfileFieldType) { + // If a type is specified, use that instead of the type implied by the name + type = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } } + } } @@ -128,17 +144,31 @@ public class QueryProfileProperties extends Properties { } } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + e.getMessage()); // TODO: Nest instead + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); } } @Override - public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, + public void clearAll(CompoundName name, Map<String, String> context) { + if (references == null) + references = new ArrayList<>(); + references.add(new Pair<>(name, null)); + + if (values != null) + values.keySet().removeIf(key -> key.hasPrefix(name)); + } + + @Override + public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { path = unalias(path, context); if (context == null) context = Collections.emptyMap(); - Map<String, Object> properties = profile.listValues(path, context, substitution); + Map<String, Object> properties = new HashMap<>(); + for (var entry : profile.listValues(path, context, substitution).entrySet()) { + if (references != null && containsNullParentOf(path, references)) continue; + properties.put(entry.getKey(), entry.getValue()); + } properties.putAll(super.listProperties(path, context, substitution)); if (references != null) { @@ -155,8 +185,14 @@ public class QueryProfileProperties extends Properties { pathInReference = path.rest(refEntry.getFirst().size()); prefixToReferenceKeys = CompoundName.empty; } - for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { - properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + if (refEntry.getSecond() == null) { + if (refEntry.getFirst().hasPrefix(path)) + properties.put(prefixToReferenceKeys.toString(), null); + } + else { + for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { + properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + } } } @@ -231,6 +267,12 @@ public class QueryProfileProperties extends Properties { return null; } + private boolean containsNullParentOf(CompoundName path, List<Pair<CompoundName, CompiledQueryProfile>> properties) { + if (properties.contains(new Pair<>(path, (CompiledQueryProfile)null))) return true; + if (path.size() > 0 && containsNullParentOf(path.first(path.size() - 1), properties)) return true; + return false; + } + CompoundName unalias(CompoundName name, Map<String,String> context) { if (profile.getTypes().isEmpty()) return name; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 644d366e7d0..94f4e4747a0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -134,7 +134,6 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) { Map<String, Object> values = new HashMap<>(); for (Map.Entry<CompoundName, DimensionalValue<ValueWithSource>> entry : entries.entrySet()) { - if ( entry.getKey().size() <= prefix.size()) continue; if ( ! entry.getKey().hasPrefix(prefix)) continue; ValueWithSource valueWithSource = entry.getValue().get(context); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index 33f07a58195..1b1cdce5890 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -33,7 +33,7 @@ public class QueryProfileXMLReader { * Reads all query profile xml files in a given directory, * and all type xml files from the immediate subdirectory "types/" (if any) * - * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML + * @throws IllegalArgumentException if the directory is not readable, or if there is some error in the XML */ public QueryProfileRegistry read(String directory) { List<NamedReader> queryProfileReaders = new ArrayList<>(); @@ -58,7 +58,7 @@ public class QueryProfileXMLReader { return read(queryProfileTypeReaders,queryProfileReaders); } catch (IOException e) { - throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e); + throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'", e); } finally { closeAll(queryProfileReaders); @@ -105,14 +105,14 @@ public class QueryProfileXMLReader { "' must be 'query-profile-type', not '" + root.getNodeName() + "'"); } - String idString=root.getAttribute("id"); + String idString = root.getAttribute("id"); if (idString == null || idString.equals("")) throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element"); ComponentId id = new ComponentId(idString); - validateFileNameToId(reader.getName(),id,"query profile type"); + validateFileNameToId(reader.getName(), id,"query profile type"); QueryProfileType type = new QueryProfileType(id); - type.setMatchAsPath(XML.getChild(root,"match") != null); - type.setStrict(XML.getChild(root,"strict") != null); + type.setMatchAsPath(XML.getChild(root, "match") != null); + type.setStrict(XML.getChild(root, "strict") != null); registry.register(type); queryProfileTypeElements.add(root); } @@ -145,7 +145,7 @@ public class QueryProfileXMLReader { queryProfile.setType(type); } - Element dimensions = XML.getChild(root,"dimensions"); + Element dimensions = XML.getChild(root, "dimensions"); if (dimensions != null) queryProfile.setDimensions(toArray(XML.getValue(dimensions))); @@ -215,7 +215,7 @@ public class QueryProfileXMLReader { try { String fieldTypeName = field.getAttribute("type"); if (fieldTypeName == null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute"); - FieldType fieldType=FieldType.fromString(fieldTypeName,registry); + FieldType fieldType = FieldType.fromString(fieldTypeName, registry); type.addField(new FieldDescription(name, fieldType, field.getAttribute("alias"), diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index b8290fa092b..6c30f1a8b05 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -97,7 +97,7 @@ public class FieldDescription implements Comparable<FieldDescription> { this.type = type; // Forbidden until we can figure out the right semantics - if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names"); + if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names"); this.aliases = ImmutableList.copyOf(aliases); this.mandatory = mandatory; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 07c9e4475ec..c02aada2062 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.FreezableSimpleComponent; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.OverridableQueryProfile; import com.yahoo.search.query.profile.QueryProfile; import java.util.ArrayList; @@ -24,6 +25,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; public class QueryProfileType extends FreezableSimpleComponent { private final CompoundName componentIdAsCompoundName; + /** The fields of this query profile type */ private Map<String, FieldDescription> fields; @@ -217,25 +219,38 @@ public class QueryProfileType extends FreezableSimpleComponent { /** Returns the type of the given query profile type declared as a field in this */ public QueryProfileType getType(String localName) { - FieldDescription fieldDescription=getField(localName); - if (fieldDescription ==null) return null; + FieldDescription fieldDescription = getField(localName); + if (fieldDescription == null) return null; if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null; return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); } + /** Returns the field type of the given name under this, of null if none */ + public FieldType getFieldType(CompoundName name) { + FieldDescription field = getField(name.first()); + if (field == null) return null; + + FieldType fieldType = field.getType(); + if (name.size() == 1) return fieldType; + + if ( ! (fieldType instanceof QueryProfileFieldType)) return null; + + return ((QueryProfileFieldType)fieldType).getQueryProfileType().getFieldType(name.rest()); + } + /** * Returns the description of the field with the given name in this type or an inherited type * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile. */ public FieldDescription getField(String name) { - FieldDescription field=fields.get(name); - if ( field!=null ) return field; + FieldDescription field = fields.get(name); + if ( field != null ) return field; if ( isFrozen() ) return null; // Inherited are collapsed into this for (QueryProfileType inheritedType : this.inherited() ) { - field=inheritedType.getField(name); - if (field!=null) return field; + field = inheritedType.getField(name); + if (field != null) return field; } return null; @@ -276,7 +291,7 @@ public class QueryProfileType extends FreezableSimpleComponent { // Add (/to) a query profile type containing the rest of the name. // (we do not need the field description settings for intermediate query profile types // as the leaf entry will enforce them) - QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry); + QueryProfileType type = extendOrCreateQueryProfileType(name.first(), registry); type.addField(fieldDescription.withName(name.rest()), registry); } else { @@ -288,27 +303,42 @@ public class QueryProfileType extends FreezableSimpleComponent { addAlias(alias, fieldDescription.getName()); } - private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + private QueryProfileType extendOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + QueryProfileType type = null; FieldDescription fieldDescription = getField(name); if (fieldDescription != null) { - if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType)) + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " + "already a " + fieldDescription.getType()); QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType(); - QueryProfileType type = fieldType.getQueryProfileType(); - if (type == null) { // an as-yet untyped reference; add type - type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type))); - } - return type; + type = fieldType.getQueryProfileType(); + } + + if (type == null) { + type = registry.getComponent(name); + } + + // found in registry but not already added in *this* type (getField also checks parents): extend it + if (type != null && ! fields.containsKey(name)) { + type = new QueryProfileType(ComponentId.createAnonymousComponentId(type.getIdString()), + new HashMap<>(), + List.of(type)); + } + + if (type == null) { // create it + type = new QueryProfileType(ComponentId.createAnonymousComponentId(name)); + } + + if (fieldDescription == null) { + fieldDescription = new FieldDescription(name, new QueryProfileFieldType(type)); } else { - QueryProfileType type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type))); - return type; + fieldDescription = fieldDescription.withType(new QueryProfileFieldType(type)); } + + registry.register(type); + fields.put(name, fieldDescription); + return type; } private void addAlias(String alias, String field) { @@ -362,6 +392,7 @@ public class QueryProfileType extends FreezableSimpleComponent { return other.getId().equals(this.getId()); } + @Override public String toString() { return "query profile type '" + getId() + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 30fc98ac6b1..643e215daef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -26,7 +26,10 @@ public class PropertyMap extends Properties { /** The properties of this */ private Map<CompoundName, Object> properties = new LinkedHashMap<>(); - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { + if (value == null) // Both clear and forward + properties.remove(name); + if (shouldSet(name, value)) properties.put(name, value); else @@ -37,7 +40,7 @@ public class PropertyMap extends Properties { * Return true if this value should be set in this map, false if the set should be propagated instead * This default implementation always returns true. */ - protected boolean shouldSet(CompoundName name,Object value) { return true; } + protected boolean shouldSet(CompoundName name, Object value) { return true; } @Override public Object get(CompoundName name, Map<String,String> context, diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index a4c150b606e..96f73e925af 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -44,7 +44,7 @@ public class QueryProperties extends Properties { @Override public Object get(CompoundName key, - Map<String,String> context, + Map<String, String> context, com.yahoo.processing.request.Properties substitution) { if (key.size() == 2 && key.first().equals(Model.MODEL)) { Model model = query.getModel(); @@ -294,7 +294,7 @@ public class QueryProperties extends Properties { super.set(key,value,context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation - if (e.getMessage().startsWith("Could not set")) + if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) throw e; else throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index ee09521fa74..6cf27fc9a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -7,7 +7,7 @@ import com.yahoo.search.query.Properties; import java.util.Map; /** - * Turns get(name) into get(name,request) using the request given at construction time. + * Turns get(name) into get(name, request) using the request given at construction time. * This is used to allow the query's request to be supplied to all property requests * without forcing users of the query.properties() to supply this explicitly. * @@ -22,18 +22,18 @@ public class RequestContextProperties extends Properties { } @Override - public Object get(CompoundName name,Map<String,String> context, + public Object get(CompoundName name, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.get(name, context == null ? requestMap : context, substitution); } @Override - public void set(CompoundName name,Object value,Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String,String> context) { super.set(name, value, context == null ? requestMap : context); } @Override - public Map<String, Object> listProperties(CompoundName path,Map<String,String> context, + public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.listProperties(path, context == null ? requestMap : context, substitution); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 1e8f436a05a..25488aa7bbc 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -44,4 +44,5 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { Index index = indexFacts.getIndex(sb.toString()); return index.isLowercase() || index.isAttribute(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index af453983f89..31f8194b3b7 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; +import com.yahoo.container.logging.TraceRenderer; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Inspector; @@ -44,8 +45,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; import org.json.JSONArray; import org.json.JSONObject; @@ -111,10 +110,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final String ROOT = "root"; private static final String SOURCE = "source"; private static final String TOTAL_COUNT = "totalCount"; - private static final String TRACE = "trace"; - private static final String TRACE_CHILDREN = "children"; - private static final String TRACE_MESSAGE = "message"; - private static final String TRACE_TIMESTAMP = "timestamp"; private static final String TIMING = "timing"; private static final String QUERY_TIME = "querytime"; private static final String SUMMARY_FETCH_TIME = "summaryfetchtime"; @@ -132,145 +127,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private LongSupplier timeSource; private OutputStream stream; - private class TraceRenderer extends TraceVisitor { - private final long basetime; - private boolean hasFieldName = false; - int emittedChildNesting = 0; - int currentChildNesting = 0; - private boolean insideOpenObject = false; - - TraceRenderer(long basetime) { - this.basetime = basetime; - } - - @Override - public void entering(TraceNode node) { - ++currentChildNesting; - } - - @Override - public void leaving(TraceNode node) { - conditionalEndObject(); - if (currentChildNesting == emittedChildNesting) { - try { - generator.writeEndArray(); - generator.writeEndObject(); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - --emittedChildNesting; - } - --currentChildNesting; - } - - @Override - public void visit(TraceNode node) { - try { - doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext()); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { - boolean dirty = false; - if (timestamp != 0L) { - header(); - generator.writeStartObject(); - generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime); - dirty = true; - } - if (payload != null) { - if (!dirty) { - header(); - generator.writeStartObject(); - } - generator.writeFieldName(TRACE_MESSAGE); - fieldConsumer.renderFieldContentsDirect(payload); - dirty = true; - } - if (dirty) { - if (!hasChildren) { - generator.writeEndObject(); - } else { - setInsideOpenObject(true); - } - } - } - - private void header() { - fieldName(); - for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) { - startChildArray(); - } - emittedChildNesting = currentChildNesting; - } - - private void startChildArray() { - try { - conditionalStartObject(); - generator.writeArrayFieldStart(TRACE_CHILDREN); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void conditionalStartObject() throws IOException { - if (!isInsideOpenObject()) { - generator.writeStartObject(); - } else { - setInsideOpenObject(false); - } - } - - private void conditionalEndObject() { - if (isInsideOpenObject()) { - // This triggers if we were inside a data node with payload and - // subnodes, but none of the subnodes contained data - try { - generator.writeEndObject(); - setInsideOpenObject(false); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - } - - private void fieldName() { - if (hasFieldName) { - return; - } - - try { - generator.writeFieldName(TRACE); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - hasFieldName = true; - } - - boolean isInsideOpenObject() { - return insideOpenObject; - } - - void setInsideOpenObject(boolean insideOpenObject) { - this.insideOpenObject = insideOpenObject; - } - } - - private static final class TraceRenderWrapper extends RuntimeException { - - /** - * Should never be serialized, but this is still needed. - */ - private static final long serialVersionUID = 2L; - - TraceRenderWrapper(IOException wrapped) { - super(wrapped); - } - - } - public JsonRenderer() { this(null); } @@ -352,8 +208,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { long basetime = trace.traceNode().timestamp(); if (basetime == 0L) basetime = getResult().getElapsedTime().first(); - trace.accept(new TraceRenderer(basetime)); - } catch (TraceRenderWrapper e) { + trace.accept(new TraceRenderer(generator, fieldConsumer, basetime)); + } catch (TraceRenderer.TraceRenderWrapper e) { throw new IOException(e); } } @@ -641,11 +497,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private String getJsonCallback() { Result result = getResult(); - if (result != null) { - Query query = result.getQuery(); - if (query != null) { - return query.properties().getString(JSON_CALLBACK, null); - } + Query query = result.getQuery(); + if (query != null) { + return query.properties().getString(JSON_CALLBACK, null); } return null; } @@ -671,7 +525,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * This instance is reused for all hits of a Result since we are in a single-threaded context * and want to limit object creation. */ - public static class FieldConsumer implements Hit.RawUtf8Consumer { + public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer { private final JsonGenerator generator; private final boolean debugRendering; @@ -788,11 +642,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { if (field instanceof Inspectable && ! (field instanceof FeatureData)) { renderInspector(((Inspectable)field).inspect()); } else { - renderFieldContentsDirect(field); + accept(field); } } - private void renderFieldContentsDirect(Object field) throws IOException { + @Override + public void accept(Object field) throws IOException { if (field == null) { generator.writeNull(); } else if (field instanceof Boolean) { diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 5cc34ff5b28..84fe88d0292 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -3,7 +3,6 @@ package com.yahoo.search.searchchain; import com.yahoo.component.chain.Chain; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; @@ -11,7 +10,6 @@ import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; -import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; diff --git a/container-search/src/main/resources/configdefinitions/strict-contracts.def b/container-search/src/main/resources/configdefinitions/strict-contracts.def index f9dd788c054..5ceb37db8d1 100644 --- a/container-search/src/main/resources/configdefinitions/strict-contracts.def +++ b/container-search/src/main/resources/configdefinitions/strict-contracts.def @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=search.federation +## DEPRECATED: This config will be removed on Vespa 8 ## A config to control whether to activate strict adherence to public contracts ## in the container. Usually, the container tries to do a best effort of hiding ## some undesirable effects of the the public contracts. Modifying this config @@ -11,11 +12,9 @@ namespace=search.federation ## can be construed to be unnecessary. searchchains bool default=false -# WARNING: Beta feature, might be removed soon. -# Propagate source.(sourceName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES} and -# provider.(providerName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES} -# to the outgoing query. -# All means all in QueryProperties.PER_SOURCE_QUERY_PROPERTIES -# OFFSET_HITS means {Query.HITS, Query.OFFSET} -# NONE means {} -propagateSourceProperties enum {ALL, OFFSET_HITS, NONE} default=ALL +# EVERY, // Propagate any property starting by source.[sourceName] and provider.[providerName] +# NATIVE, // Propagate native properties only +# ALL, // Deprecated synonym of NATIVE +# OFFSET_HITS, // Propagate offset ands hits only +# NONE // propagate no properties +propagateSourceProperties enum {EVERY, NATIVE, ALL, OFFSET_HITS, NONE} default=EVERY diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5b6a4b68930..8b7a57c38e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; @@ -514,8 +515,10 @@ public class ClusterSearcherTestCase { DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); - Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"), - new DispatchConfig.Builder().build(), + DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); + Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig), + ComponentId.createAnonymousComponentId("test-id"), + dispatchConfig, createClusterInfoConfig(), vipStatus, new MockMetric()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index b5e54b46e5a..63475c9c189 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -14,7 +14,6 @@ import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -151,7 +150,8 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - Dispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); + dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); dispatch_1.deconstruct(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 440e3b8d78f..2a2c8410b2c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -2,8 +2,10 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.container.handler.VipStatus; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; @@ -13,25 +15,27 @@ import java.util.List; class MockDispatcher extends Dispatcher { + public final ClusterMonitor clusterMonitor; + public static MockDispatcher create(List<Node> nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, rpcResourcePool, 1, new VipStatus()); + return create(nodes, rpcResourcePool, new VipStatus()); } - public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus); - return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool); + var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool)); + return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { - this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { + this(clusterMonitor, searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { - super(searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { + super(clusterMonitor, searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + this.clusterMonitor = clusterMonitor; } static DispatchConfig toDispatchConfig(List<Node> nodes) { diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java index 11922cf640a..36137abd9b8 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java @@ -1,15 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform.test; +import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.prelude.querytransform.RecallSearcher; import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -38,9 +45,17 @@ public class QueryRewriteTestCase { assertRewritten(query, "OR sddocname:per foo bar"); ((OrItem)query.getModel().getQueryTree().getRoot()).getItem(2).setRanked(false); // set 'bar' unranked assertRewritten(query, "OR sddocname:per foo"); - assertRewritten("sddocname:per OR foo OR (bar AND fuz)", "per", "OR sddocname:per foo (AND bar fuz)"); + } + @Test + public void testRankContributingTermsAreNotRemovedOnFullRecall() { + Query query = new Query(QueryTestCase.httpEncode("?query=default:term1 OR default:term2 OR default:term3 OR sddocname:per&type=adv&recall=+id:1&restrict=per")); + RecallSearcher searcher = new RecallSearcher(); + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertNull(result.hits().getError()); + assertNull(QueryCanonicalizer.canonicalize(query)); + assertRewritten(query, "AND (OR default:term1 default:term2 default:term3 sddocname:per) |id:1"); } @Test @@ -88,6 +103,7 @@ public class QueryRewriteTestCase { private static void assertRewritten(Query query, String expectedOptimizedQuery) { QueryRewrite.optimizeByRestrict(query); + QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 9da1c184505..0086f1b3571 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -108,7 +108,7 @@ public class BlendingSearcherTestCase { entry.getValue())); } - StrictContractsConfig contracts = new StrictContractsConfig(new StrictContractsConfig.Builder()); + StrictContractsConfig contracts = new StrictContractsConfig.Builder().build(); FederationSearcher fedSearcher = new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); @@ -124,7 +124,6 @@ public class BlendingSearcherTestCase { @Test public void testitTwoPhase() { - DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); DocumentSourceSearcher chain3 = new DocumentSourceSearcher(); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 65011ffb562..f4bf957e29a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -52,6 +52,7 @@ public class ValidateSortingSearcherTestCase { assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[rank]")); assertEquals("[ASCENDING:[docid]]", quoteAndTransform("+[docid]")); assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevancy]")); + assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevance]")); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index de6bafa267a..5433a28dd6e 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -1,22 +1,21 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.prelude.fastsearch.test.MockMetric; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import org.junit.Test; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.Callable; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.junit.Assert.assertEquals; @@ -39,9 +38,10 @@ public class DispatcherTest { assertEquals(2, nodes.get(0).key()); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(q, null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -53,9 +53,10 @@ public class DispatcherTest { } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -69,9 +70,10 @@ public class DispatcherTest { assertTrue(acceptIncompleteCoverage); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -80,8 +82,9 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 1); MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); disp.getSearchInvoker(new Query(), null); + disp.deconstruct(); fail("Expected exception"); } catch (IllegalStateException e) { @@ -142,7 +145,7 @@ public class DispatcherTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { fail("Unexpected call to createPinger"); return null; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 0496194f8ed..36b476e2936 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index a976b287f63..32c6738fc3b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, 1, null); + super(clusterId, dispatchConfig, null, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 10a579b0e4f..cf90a1c6d81 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -14,12 +14,12 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +34,7 @@ public class SearchClusterTest { final int nodesPerGroup; final VipStatus vipStatus; final SearchCluster searchCluster; + final ClusterMonitor clusterMonitor; final List<AtomicInteger> numDocsPerNode; List<AtomicInteger> pingCounts; @@ -57,74 +58,76 @@ public class SearchClusterTest { numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } - searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, vipStatus); + searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, + vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); + clusterMonitor = new ClusterMonitor(searchCluster, false); + searchCluster.addMonitoring(clusterMonitor); } - void startMonitoring() { - searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); - } - - static private int maxFrom(List<AtomicInteger> list) { - int max = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() > max) { - max = v.get(); + private int maxPingCount() { + int max = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() > max) { + max = count.get(); } } return max; } - private static int minFrom(List<AtomicInteger> list) { - int min = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() < min) { - min = v.get(); + private int minPingCount() { + int min = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() < min) { + min = count.get(); } } return min; } - private void waitAtLeast(int atLeast, List<AtomicInteger> list) { - while (minFrom(list) < atLeast) { + void waitOneFullPingRound() { + int minPingCount = minPingCount(); + int atLeast = maxPingCount() + 1; + while (minPingCount < atLeast) { ExecutorService executor = Executors.newCachedThreadPool(); - searchCluster.clusterMonitor().ping(executor); + clusterMonitor.ping(executor); executor.shutdown(); try { boolean completed = executor.awaitTermination(120, TimeUnit.SECONDS); if ( ! completed ) throw new IllegalStateException("Ping thread timed out"); + // Since a separate thread will be modifying values in pingCounts, we need to wait for the thread to + // finish before re-reading the minimum value + minPingCount = minPingCount(); } catch (InterruptedException e) { - System.out.println("Ping thread interrupted"); + throw new RuntimeException(e); } } } - void waitOneFullPingRound() { - waitAtLeast(maxFrom(pingCounts) + 1, pingCounts); - } - @Override public void close() { - searchCluster.shutDown(); + clusterMonitor.shutdown(); } static class Factory implements PingFactory { - static class Pinger implements Callable<Pong> { + static class PingJob implements Pinger { private final AtomicInteger numDocs; private final AtomicInteger pingCount; - Pinger(AtomicInteger numDocs, AtomicInteger pingCount) { + private final PongHandler pongHandler; + PingJob(AtomicInteger numDocs, AtomicInteger pingCount, PongHandler pongHandler) { this.numDocs = numDocs; this.pingCount = pingCount; + this.pongHandler = pongHandler; } @Override - public Pong call() { + public void ping() { int docs = numDocs.get(); - pingCount.incrementAndGet(); - return (docs < 0) + pongHandler.handle ((docs < 0) ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs)) - : new Pong(docs); + : new Pong(docs)); + pingCount.incrementAndGet(); } } @@ -139,9 +142,9 @@ public class SearchClusterTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { int index = node.group() * numPerGroup + node.key(); - return new Pinger(activeDocs.get(index), pingCounts.get(index)); + return new PingJob(activeDocs.get(index), pingCounts.get(index), pongHandler); } } @@ -153,7 +156,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); } @@ -162,7 +164,6 @@ public class SearchClusterTest { @Test public void requireThatZeroDocsAreFine() { try (State test = new State("cluster.1", 2, "a", "b")) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -184,7 +185,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); } @@ -196,7 +196,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); test.numDocsPerNode.get(0).set(-1); @@ -209,7 +208,6 @@ public class SearchClusterTest { public void requireThatVipStatusDownWhenLocalIsDown() { try (State test = new State("cluster.1",1,HostName.getLocalhost(), "b")) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); @@ -245,7 +243,6 @@ public class SearchClusterTest { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); @@ -273,8 +270,8 @@ public class SearchClusterTest { static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) { List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup); for (int g = 0; g < numGroups; g++) { - for (int n=0; n < nodesPerGroup; n++) { - nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString()); + for (int n = 0; n < nodesPerGroup; n++) { + nodeNames.add("node." + g + '.' + n); } } return nodeNames; @@ -284,7 +281,6 @@ public class SearchClusterTest { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); @@ -310,4 +306,18 @@ public class SearchClusterTest { verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3); } + @Test + public void requireThatPingSequenceIsUpHeld() { + Node node = new Node(1, "n", 1); + assertEquals(1, node.createPingSequenceId()); + assertEquals(2, node.createPingSequenceId()); + assertEquals(0, node.getLastReceivedPongId()); + assertTrue(node.isLastReceivedPong(2)); + assertEquals(2, node.getLastReceivedPongId()); + assertFalse(node.isLastReceivedPong(1)); + assertFalse(node.isLastReceivedPong(2)); + assertTrue(node.isLastReceivedPong(3)); + assertEquals(3, node.getLastReceivedPongId()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java index 111b7a8eb69..65cb4dff1f8 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java @@ -201,13 +201,33 @@ public class FederationSearcherTestCase { } @Test - public void testPropertyPropagation() { - Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL); + public void testPropertyPropagation_native() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.NATIVE); assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary()); assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); + assertEquals(null, result.hits().get(0).getQuery().properties().get("custom")); + } + + @Test + public void testPropertyPropagation_every() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.EVERY); + + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary()); + assertEquals("foo", result.hits().get(0).getQuery().properties().get("customSourceProperty")); + assertEquals(null, result.hits().get(1).getQuery().properties().get("customSourceProperty")); + assertEquals(null, result.hits().get(0).getQuery().properties().get("custom.source.property")); + assertEquals("bar", result.hits().get(1).getQuery().properties().get("custom.source.property")); + assertEquals(13, result.hits().get(0).getQuery().properties().get("hits")); + assertEquals(1, result.hits().get(0).getQuery().properties().get("offset")); + assertEquals(10, result.hits().get(1).getQuery().properties().get("hits")); + assertEquals(0, result.hits().get(1).getQuery().properties().get("offset")); + + assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); } private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { @@ -215,7 +235,7 @@ public class FederationSearcherTestCase { addChained(new MockSearcher(), "mySource2"); Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); - Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle")); + Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle&source.mySource1.customSourceProperty=foo&source.mySource2.custom.source.property=bar&source.mySource1.hits=13&source.mySource1.offset=1")); Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); assertNull(result.hits().getError()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java index 326e37ede38..6d9c2218022 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit; */ public class GroupingParserBenchmarkTest { - private static final int NUM_RUNS = 10;//000; + private static final int NUM_RUNS = 10; private static final Map<String, Long> PREV_RESULTS = new LinkedHashMap<>(); static { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index cd080405a7d..c6686471dc8 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -247,6 +247,7 @@ public class GroupingParserTestCase { assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))"); assertParse("all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))"); assertParse("all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>)))"); assertParse("all(group(predefined(foo, bucket[1, 2>)))"); assertParse("all(group(predefined(foo, bucket[-1, 2>)))"); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 02e2152d7c9..272092b6fc0 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -12,7 +12,7 @@ import com.yahoo.net.HostName; import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.After; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index 0f4a22ef368..e9f7ff24d42 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -384,43 +384,64 @@ public class XmlReadingTestCase { assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } @Test public void testRefOverrideTyped() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); { // Original reference - Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile1",query.properties().get("profileRef.name")); - assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile1", query.properties().get("profileRef.name")); + assertEquals("myProfile1Only", query.properties().get("profileRef.myProfile1Only")); assertNull(query.properties().get("profileRef.myProfile2Only")); } { // Overridden reference - Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); - assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); + assertEquals("myProfile2Only", query.properties().get("profileRef.myProfile2Only")); assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } + @Test + public void testTensorTypes() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/tensortypes").compile(); + + QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1"); + assertEquals("tensor<float>(x[1])", type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).stringValue()); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)"))); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)"))); + + QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2"); + assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)"))); + assertEquals("tensor<float>(x[2])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).stringValue()); + assertEquals("tensor<float>(x[3])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).stringValue()); + + Query queryProfile1 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile1")); + assertEquals("Is received as a tensor tensor", "tensor<float>(x[1]):[1.2]", queryProfile1.properties().get("ranking.features.query(tensor_1)").toString()); + + Query queryProfile2 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile2")); + assertEquals("Is received as a string", "[1.200]", queryProfile2.properties().get("ranking.features.query(tensor_1)").toString()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml new file mode 100644 index 00000000000..000fd3e1c5b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml @@ -0,0 +1,2 @@ +<query-profile id="profile1" type="type1"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml new file mode 100644 index 00000000000..f6539da23e8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml @@ -0,0 +1,2 @@ +<query-profile id="profile2" type="type2"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml new file mode 100644 index 00000000000..3dfaab9c5f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml @@ -0,0 +1,3 @@ +<query-profile-type id="type1"> + <field name="ranking.features.query(tensor_1)" type="tensor<float>(x[1])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml new file mode 100644 index 00000000000..ed7cf23e464 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml @@ -0,0 +1,4 @@ +<query-profile-type id="type2"> + <field name="ranking.features.query(tensor_2)" type="tensor<float>(x[2])" /> + <field name="ranking.features.query(tensor_3)" type="tensor<float>(x[3])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 46efb736918..eb1584efe84 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -326,14 +326,14 @@ public class QueryProfileTestCase { assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b")); } - /** Tests that dots are followed when setting overridability */ + /** Dots are followed when setting overridability */ @Test public void testInstanceOverridable() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); profile.set("model.defaultIndex","default", null); profile.setOverridable("model.defaultIndex", false,null); - assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); // Parameters should be ignored Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null)); @@ -345,7 +345,7 @@ public class QueryProfileTestCase { assertEquals("de", query.getModel().getLanguage().languageCode()); } - /** Tests that dots are followed when setting overridability...also with variants */ + /** Dots are followed when setting overridability, also with variants */ @Test public void testInstanceOverridableWithVariants() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); @@ -504,7 +504,8 @@ public class QueryProfileTestCase { p.set("a","a-value", null); p.set("a.b","a.b-value", null); Map<String, Object> values = p.compile(null).listValues("a"); - assertEquals(1, values.size()); + assertEquals(2, values.size()); + p.set("a","a-value", null); assertEquals("a.b-value", values.get("b")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index c05c3589a30..3c200debcaf 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -397,7 +397,7 @@ public class QueryProfileTypeTestCase { @Test public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(type); registry.register(profile); @@ -447,25 +447,25 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); - topMap.set("subMap",subMap, registry); + QueryProfile subMap = new QueryProfile("topSubMap"); + topMap.set("subMap", subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(type); - subMap.set("typeProfile",test, registry); + subMap.set("typeProfile", test, registry); - QueryProfile myUser=new QueryProfile("myUser"); + QueryProfile myUser = new QueryProfile("myUser"); myUser.setType(user); - myUser.set("myUserString","userValue1", registry); - myUser.set("myUserInteger",442, registry); - test.set("myUserQueryProfile",myUser, registry); + myUser.set("myUserString", "userValue1", registry); + myUser.set("myUserInteger", 442, registry); + test.set("myUserQueryProfile", myUser, registry); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(topMap); registry.register(subMap); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 84565472820..34c3da395b7 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.test; -import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; @@ -31,12 +30,10 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.grouping.GroupingQueryParser; -import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.SessionId; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.Hit; @@ -49,23 +46,19 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -240,10 +233,10 @@ public class QueryTestCase { @Test public void test_that_cloning_preserves_timeout() { Query original = new Query(); - original.setTimeout(9876l); + original.setTimeout(9876L); Query clone = original.clone(); - assertThat(clone.getTimeout(), is(9876l)); + assertEquals(9876L, clone.getTimeout()); } @Test @@ -297,9 +290,7 @@ public class QueryTestCase { fail("Above statement should throw"); } catch (QueryException e) { // As expected. - assertThat( - Exceptions.toMessageString(e), - containsString("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); + assertTrue(Exceptions.toMessageString(e).contains("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); } } @@ -356,6 +347,61 @@ public class QueryTestCase { } @Test + public void testQueryProfileClearAndSet() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("b", "b-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + + q.properties().set("b", null, null); + assertContains(q.properties().listProperties("b"), (Object)null); + + q.properties().set("b", "b-value", null); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + } + + @Test + public void testQueryProfileClearValue() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("a", "a-value", null); + profile.set("b", "b-value", null); + profile.set("b.c", "b.c-value", null); + profile.set("b.d", "b.d-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("a-value", q.properties().get("a")); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value"); + + q.properties().set("a", null, null); + assertEquals(null, q.properties().get("a")); + + q.properties().set("b", null, null); + assertEquals(null, q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), null, "b.c-value", "b.d-value"); + + q.properties().set("b", "b-value", null); + q.properties().set("b.e", "b.e-value", null); + q.properties().set("b.f", "b.f-value", null); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.e-value", q.properties().get("b.e")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value", "b.e-value", "b.f-value"); + + q.properties().clearAll("b"); + assertEquals(null, q.properties().get("b")); + assertEquals(null, q.properties().get("b.c")); + assertEquals(null, q.properties().get("b.d")); + assertEquals(null, q.properties().get("b.e")); + assertEquals(null, q.properties().get("b.f")); + assertContains(q.properties().listProperties("b"), (Object)null); + } + + @Test public void testNotEqual() { Query q = new Query("/?query=something+test&nocache"); Query p = new Query("/?query=something+test"); @@ -900,7 +946,7 @@ public class QueryTestCase { @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -910,12 +956,12 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND myfield:'it s' myfield:\"a b\" myfield:c", query.getModel().getQueryTree().toString()); } @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -925,7 +971,7 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); } @Test @@ -996,6 +1042,19 @@ public class QueryTestCase { assertEquals(expectedDetectionText, mockLinguistics.detector.lastDetectionText); } + private void assertContains(Map<String, Object> properties, Object ... expectedValues) { + if (expectedValues == null) { + assertEquals(1, properties.size()); + assertTrue("Contains value null", properties.containsValue(null)); + } + else { + assertEquals(properties + " contains values " + Arrays.toString(expectedValues), + expectedValues.length, properties.size()); + for (Object expectedValue : expectedValues) + assertTrue("Contains value " + expectedValue, properties.containsValue(expectedValue)); + } + } + /** A linguistics instance which records the last language detection text passed to it */ private static class MockLinguistics extends SimpleLinguistics { diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 261069ea1c3..7b1b4fe6362 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -652,7 +652,6 @@ public class SelectTestCase { assertGrouping(expected, parseGrouping(grouping)); } - @Test public void testMultipleGroupings() { String grouping = "[ { \"all\" : { \"group\" : \"a\", \"each\" : { \"output\" : \"count()\"}}}, { \"all\" : { \"group\" : \"b\", \"each\" : { \"output\" : \"count()\"}}} ]"; @@ -661,6 +660,20 @@ public class SelectTestCase { assertGrouping(expected, parseGrouping(grouping)); } + @Test + public void testGroupingWithPredefinedBuckets() { + String grouping = "[ { \"all\" : { \"group\" : { \"predefined\" : [ \"foo\", { \"bucket\": [1,2]}, { \"bucket\": [3,4]} ] } } } ]"; + String expected = "[[]all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>)))]"; + assertGrouping(expected, parseGrouping(grouping)); + } + + @Test + public void testMultipleOutputs() { + String grouping = "[ { \"all\" : { \"group\" : \"b\", \"each\" : {\"output\": [ \"count()\", \"avg(foo)\" ] } } } ]"; + String expected = "[[]all(group(b) each(output(count(), avg(foo))))]"; + assertGrouping(expected, parseGrouping(grouping)); + } + //------------------------------------------------------------------- Other tests @Test @@ -763,7 +776,6 @@ public class SelectTestCase { } private List<VespaGroupingStep> parseGrouping(String grouping) { - return parser.getGroupingSteps(grouping); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java index 5c279547e17..d33f9a45c82 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; @@ -35,23 +36,31 @@ public class InstanceInformation { public String cluster; public boolean tls; public URI url; + public String scope; + public RoutingMethod routingMethod; @JsonCreator public Endpoint(@JsonProperty("cluster") String cluster , @JsonProperty("tls") boolean tls, - @JsonProperty("url") URI url) { + @JsonProperty("url") URI url, + @JsonProperty("scope") String scope, + @JsonProperty("routingMethod") RoutingMethod routingMethod) { this.cluster = cluster; this.tls = tls; this.url = url; + this.scope = scope; + this.routingMethod = routingMethod; } @Override public String toString() { - return "Endpoint {" + - "cluster=" + cluster+ - ", tls='" + tls + '\'' + - ", url='" + url+ '\'' + - '}'; + return "Endpoint{" + + "cluster='" + cluster + '\'' + + ", tls=" + tls + + ", url=" + url + + ", scope='" + scope + '\'' + + ", routingMethod=" + routingMethod + + '}'; } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 6ca5cae0455..9e5b01a91d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -46,7 +46,7 @@ public interface ServiceRegistry { Mailer mailer(); - ApplicationCertificateProvider applicationCertificateProvider(); + EndpointCertificateProvider endpointCertificateProvider(); MeteringClient meteringService(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index ec60f11060d..53e80fcb2ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; @@ -74,6 +75,16 @@ public class ZtsClientMock implements ZtsClient { } @Override + public AthenzAccessToken getAccessToken(AthenzDomain domain) { + throw new UnsupportedOperationException(); + } + + @Override + public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) { + throw new UnsupportedOperationException(); + } + + @Override public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) { throw new UnsupportedOperationException(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java deleted file mode 100644 index 41f5b65d263..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -import java.util.Objects; - -/** - * Represents a reference to a certificate and private key. - * - * @author mortent - * @author andreer - */ -public class ApplicationCertificate { - - private final String secretsKeyNamePrefix; - - public ApplicationCertificate(String secretsKeyNamePrefix) { - this.secretsKeyNamePrefix = Objects.requireNonNull(secretsKeyNamePrefix, "secretsKeyNamePrefix must be non-null"); - } - - /** The prefix of keys identifying this certificate and its private key in a key store */ - public String secretsKeyNamePrefix() { - return secretsKeyNamePrefix; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ApplicationCertificate that = (ApplicationCertificate) o; - return Objects.equals(secretsKeyNamePrefix, that.secretsKeyNamePrefix); - } - - @Override - public int hashCode() { - return Objects.hash(secretsKeyNamePrefix); - } - - @Override - public String toString() { - return "application certificate '" + secretsKeyNamePrefix + "'"; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java new file mode 100644 index 00000000000..0aa0df8ae2b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -0,0 +1,89 @@ +package com.yahoo.vespa.hosted.controller.api.integration.certificates; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * This class is used for metadata about an application's endpoint certificate on the controller. + * <p> + * It has more properties than com.yahoo.config.model.api.EndpointCertificateMetadata. + * + * @author andreer + */ +public class EndpointCertificateMetadata { + + private final String keyName; + private final String certName; + private final int version; + private final Optional<String> request_id; + private final Optional<List<String>> requestedDnsSans; + + public EndpointCertificateMetadata(String keyName, String certName, int version) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + this.request_id = Optional.empty(); + this.requestedDnsSans = Optional.empty(); + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + this.request_id = request_id; + this.requestedDnsSans = requestedDnsSans; + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { + this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans)); + } + + public String keyName() { + return keyName; + } + + public String certName() { + return certName; + } + + public int version() { + return version; + } + + public Optional<String> request_id() { + return request_id; + } + + public Optional<List<String>> requestedDnsSans() { + return requestedDnsSans; + } + + @Override + public String toString() { + return "EndpointCertificateMetadata{" + + "keyName='" + keyName + '\'' + + ", certName='" + certName + '\'' + + ", version=" + version + + ", request_id=" + request_id + + ", requestedDnsSans=" + requestedDnsSans + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointCertificateMetadata that = (EndpointCertificateMetadata) o; + return version == that.version && + keyName.equals(that.keyName) && + certName.equals(that.certName) && + request_id.equals(that.request_id) && + requestedDnsSans.equals(that.requestedDnsSans); + } + + @Override + public int hashCode() { + return Objects.hash(keyName, certName, version, request_id, requestedDnsSans); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index cc2d08c3fcd..8e81400f3c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -12,7 +12,7 @@ import java.util.UUID; /** * @author tokle */ -public class ApplicationCertificateMock implements ApplicationCertificateProvider { +public class EndpointCertificateMock implements EndpointCertificateProvider { private final Map<ApplicationId, List<String>> dnsNames = new HashMap<>(); @@ -21,11 +21,17 @@ public class ApplicationCertificateMock implements ApplicationCertificateProvide } @Override - public ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { + public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { this.dnsNames.put(applicationId, dnsNames); - return new ApplicationCertificate(String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), - applicationId.application(), - UUID.randomUUID().toString())); + String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), + applicationId.application(), + UUID.randomUUID().toString()); + return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0); + } + + @Override + public List<EndpointCertificateMetadata> listCertificates() { + return Collections.emptyList(); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java index b6ad1701449..97d2bdb3343 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java @@ -6,12 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import java.util.List; /** - * Generates a certificate. + * Generates an endpoint certificate for an application instance. * * @author andreer */ -public interface ApplicationCertificateProvider { +public interface EndpointCertificateProvider { - ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); + EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); + List<EndpointCertificateMetadata> listCertificates(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index a009f002954..722f9c4e33b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -10,7 +10,8 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -32,7 +33,7 @@ public interface ConfigServer { } PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, - Set<ContainerEndpoint> containerEndpoints, ApplicationCertificate applicationCertificate, + Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, byte[] content); void restart(DeploymentId deployment, Optional<Hostname> hostname); @@ -43,7 +44,9 @@ public interface ConfigServer { ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); - Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); + Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath); + + String getClusterControllerStatus(DeploymentId deployment, String restPath); /** * Gets the Vespa logs of the given deployment. @@ -59,22 +62,40 @@ public interface ConfigServer { List<String> getContentClusters(DeploymentId deployment); /** - * Set new status on en endpoint in one zone. + * Set new status for a endpoint of a single deployment. + * + * @param deployment The deployment to change + * @param upstreamName The upstream to modify. Upstream name is a unique identifier for the global route of a + * deployment in the shared routing layer + * @param status The new status + */ + void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status); + + /** + * Set the new status for an entire zone. * - * @param deployment The application/zone pair - * @param endpoint The endpoint to modify - * @param status The new status with metadata + * @param zone the zone + * @param in whether to set zone status to 'in' or 'out' */ - void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status); + void setGlobalRotationStatus(ZoneId zone, boolean in); /** - * Get the endpoint status for an app in one zone + * Get the endpoint status for an app in one zone. * - * @param deployment The application/zone pair - * @param endpoint The endpoint to modify + * @param deployment The deployment to change + * @param upstreamName The upstream to query. Upstream name is a unique identifier for the global route of a + * deployment in the shared routing layer * @return The endpoint status with metadata */ - EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint); + EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String upstreamName); + + /** + * Get the status for an entire zone. + * + * @param zone the zone + * @return whether the zone status is 'in' + */ + boolean getGlobalRotationStatus(ZoneId zone); /** The node repository on this config server */ NodeRepository nodeRepository(); @@ -89,7 +110,15 @@ public interface ConfigServer { List<FlagData> listFlagData(ZoneId zone); /** Gets status for tester application */ - // TODO: Remove default implementation when implemented in internal repo - default TesterCloud.Status getTesterStatus(DeploymentId deployment) { return TesterCloud.Status.SUCCESS; } + TesterCloud.Status getTesterStatus(DeploymentId deployment); + + /** Starts tests on tester node */ + String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config); + + /** Gets log from tester node */ + List<LogEntry> getTesterLog(DeploymentId deployment, long after); + + /** Is tester node ready */ + boolean isTesterReady(DeploymentId deployment); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index d0e4aa2d7d1..d8103c864df 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -6,7 +6,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -29,6 +31,9 @@ public class Node { private final Version currentOsVersion; private final Version wantedOsVersion; private final ServiceState serviceState; + private final Optional<Instant> suspendedSince; + private final Optional<Instant> currentFirmwareCheck; + private final Optional<Instant> wantedFirmwareCheck; private final long restartGeneration; private final long wantedRestartGeneration; private final long rebootGeneration; @@ -39,11 +44,14 @@ public class Node { private final ClusterType clusterType; private final boolean wantToRetire; private final boolean wantToDeprovision; + private final Optional<TenantName> reservedTo; public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, - Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState, - long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, - int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision) { + Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, + Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, + Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, + int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision, + Optional<TenantName> reservedTo) { this.hostname = hostname; this.parentHostname = parentHostname; this.state = state; @@ -54,7 +62,10 @@ public class Node { this.wantedVersion = wantedVersion; this.currentOsVersion = currentOsVersion; this.wantedOsVersion = wantedOsVersion; + this.currentFirmwareCheck = currentFirmwareCheck; + this.wantedFirmwareCheck = wantedFirmwareCheck; this.serviceState = serviceState; + this.suspendedSince = suspendedSince; this.restartGeneration = restartGeneration; this.wantedRestartGeneration = wantedRestartGeneration; this.rebootGeneration = rebootGeneration; @@ -65,6 +76,7 @@ public class Node { this.clusterType = clusterType; this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; + this.reservedTo = reservedTo; } public HostName hostname() { @@ -105,10 +117,22 @@ public class Node { return wantedOsVersion; } + public Optional<Instant> currentFirmwareCheck() { + return currentFirmwareCheck; + } + + public Optional<Instant> wantedFirmwareCheck() { + return wantedFirmwareCheck; + } + public ServiceState serviceState() { return serviceState; } + public Optional<Instant> suspendedSince() { + return suspendedSince; + } + public long restartGeneration() { return restartGeneration; } @@ -149,6 +173,8 @@ public class Node { return wantToDeprovision; } + public Optional<TenantName> reservedTo() { return reservedTo; } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -202,7 +228,10 @@ public class Node { private Version wantedVersion; private Version currentOsVersion; private Version wantedOsVersion; + private Optional<Instant> currentFirmwareCheck = Optional.empty(); + private Optional<Instant> wantedFirmwareCheck = Optional.empty(); private ServiceState serviceState; + private Optional<Instant> suspendedSince = Optional.empty(); private long restartGeneration; private long wantedRestartGeneration; private long rebootGeneration; @@ -213,7 +242,8 @@ public class Node { private ClusterType clusterType; private boolean wantToRetire; private boolean wantToDeprovision; - + private Optional<TenantName> reservedTo = Optional.empty(); + public Builder() { } public Builder(Node node) { @@ -227,7 +257,10 @@ public class Node { this.wantedVersion = node.wantedVersion; this.currentOsVersion = node.currentOsVersion; this.wantedOsVersion = node.wantedOsVersion; + this.currentFirmwareCheck = node.currentFirmwareCheck; + this.wantedFirmwareCheck = node.wantedFirmwareCheck; this.serviceState = node.serviceState; + this.suspendedSince = node.suspendedSince; this.restartGeneration = node.restartGeneration; this.wantedRestartGeneration = node.wantedRestartGeneration; this.rebootGeneration = node.rebootGeneration; @@ -238,6 +271,7 @@ public class Node { this.clusterType = node.clusterType; this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; + this.reservedTo = node.reservedTo; } public Builder hostname(HostName hostname) { @@ -290,11 +324,26 @@ public class Node { return this; } + public Builder currentFirmwareCheck(Instant currentFirmwareCheck) { + this.currentFirmwareCheck = Optional.ofNullable(currentFirmwareCheck); + return this; + } + + public Builder wantedFirmwareCheck(Instant wantedFirmwareCheck) { + this.wantedFirmwareCheck = Optional.ofNullable(wantedFirmwareCheck); + return this; + } + public Builder serviceState(ServiceState serviceState) { this.serviceState = serviceState; return this; } + public Builder suspendedSince(Instant suspendedSince) { + this.suspendedSince = Optional.ofNullable(suspendedSince); + return this; + } + public Builder restartGeneration(long restartGeneration) { this.restartGeneration = restartGeneration; return this; @@ -345,10 +394,17 @@ public class Node { return this; } + public Builder reservedTo(TenantName tenant) { + this.reservedTo = Optional.of(tenant); + return this; + } + public Node build() { - return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, - wantedOsVersion, serviceState, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, - cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision); + return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, + currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, + suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, + cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo); } + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 6d18c75b952..dd2390a1281 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -6,12 +6,14 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -37,6 +39,8 @@ public interface NodeRepository { NodeList listNodes(ZoneId zone, ApplicationId application); + NodeList listNodes(ZoneId zone, List<HostName> hostnames); + /** List all nodes in given zone */ default List<Node> list(ZoneId zone) { return listNodes(zone).nodes().stream() @@ -45,6 +49,13 @@ public interface NodeRepository { } /** List all nodes in zone owned by given application */ + default List<Node> list(ZoneId zone, List<HostName> hostnames) { + return listNodes(zone, hostnames).nodes().stream() + .map(NodeRepository::toNode) + .collect(Collectors.toUnmodifiableList()); + } + + /** List all nodes in zone owned by given application */ default List<Node> list(ZoneId zone, ApplicationId application) { return listNodes(zone, application).nodes().stream() .map(NodeRepository::toNode) @@ -97,7 +108,10 @@ public interface NodeRepository { versionFrom(node.getWantedVespaVersion()), versionFrom(node.getCurrentOsVersion()), versionFrom(node.getWantedOsVersion()), + Optional.ofNullable(node.getCurrentFirmwareCheck()).map(Instant::ofEpochMilli), + Optional.ofNullable(node.getWantedFirmwareCheck()).map(Instant::ofEpochMilli), fromBoolean(node.getAllowedToBeDown()), + Optional.ofNullable(node.suspendedSinceMillis()).map(Instant::ofEpochMilli), toInt(node.getCurrentRestartGeneration()), toInt(node.getRestartGeneration()), toInt(node.getCurrentRebootGeneration()), @@ -107,7 +121,8 @@ public interface NodeRepository { clusterIdOf(node.getMembership()), clusterTypeOf(node.getMembership()), node.getWantToRetire(), - node.getWantToDeprovision()); + node.getWantToDeprovision(), + Optional.ofNullable(node.getReservedTo()).map(name -> TenantName.from(name))); } private static String clusterIdOf(NodeMembership nodeMembership) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 7aac0826b2b..1f546989b8d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -143,15 +143,15 @@ public enum JobType { private final String jobName; private final Map<SystemName, ZoneId> zones; - private final boolean isTest; + private final boolean isProductionTest; - JobType(String jobName, Map<SystemName, ZoneId> zones, boolean isTest) { + JobType(String jobName, Map<SystemName, ZoneId> zones, boolean isProductionTest) { if (zones.values().stream().map(ZoneId::environment).distinct().count() > 1) throw new IllegalArgumentException("All zones of a job must be in the same environment"); this.jobName = jobName; this.zones = zones; - this.isTest = isTest; + this.isProductionTest = isProductionTest; } JobType(String jobName, Map<SystemName, ZoneId> zones) { @@ -176,10 +176,10 @@ public enum JobType { public boolean isProduction() { return environment() == Environment.prod; } /** Returns whether this job runs tests */ - public boolean isTest() { return isTest; } + public boolean isTest() { return isProductionTest || environment().isTest(); } /** Returns whether this job deploys to a zone */ - public boolean isDeployment() { return ! (isProduction() && isTest); } + public boolean isDeployment() { return ! (isProduction() && isProductionTest); } /** Returns the environment of this job type, or null if it does not have an environment */ public Environment environment() { @@ -206,7 +206,7 @@ public enum JobType { /** Returns the job type for the given zone */ public static Optional<JobType> from(SystemName system, ZoneId zone) { - return from(system, zone, false); + return from(system, zone, zone.environment().isTest()); } /** Returns the production test job type for the given environment and region or null if none */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index ef97ef27a72..a1c4d379b6c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import java.net.URI; @@ -16,21 +17,38 @@ public interface TesterCloud { /** Signals the tester to run its tests. */ void startTests(URI testerUrl, Suite suite, byte[] config); + /** Signals the tester to run its tests. */ + void startTests(DeploymentId deploymentId, Suite suite, byte[] config); + /** Returns the log entries from the tester with ids after the given threshold. */ List<LogEntry> getLog(URI testerUrl, long after); + /** Returns the log entries from the tester with ids after the given threshold. */ + List<LogEntry> getLog(DeploymentId deploymentId, long after); + /** Returns the current status of the tester. */ Status getStatus(URI testerUrl); + /** Returns the current status of the tester. */ + Status getStatus(DeploymentId deploymentId); + /** Returns whether the container is ready to serve. */ boolean ready(URI endpointUrl); /** Returns whether the test container is ready to serve */ boolean testerReady(URI endpointUrl); + /** Returns whether the test container is ready to serve */ + boolean testerReady(DeploymentId deploymentId); + /** Returns whether the given URL is registered in DNS. */ boolean exists(URI endpointUrl); + /** + * Returns whether the given URL is registered in DNS. Always returns true, + * as endpoints are not use in this case + */ + default boolean exists(DeploymentId deploymentId) { return true; } enum Status { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 75d49e542dc..b54446c071e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -45,7 +45,7 @@ public class MemoryNameService implements NameService { .map(target -> new Record(Record.Type.ALIAS, name, target.asData())) .collect(Collectors.toList()); // Satisfy idempotency contract of interface - removeRecords(findRecords(Record.Type.ALIAS, name)); + removeRecords(records); records.forEach(this::add); return records; } @@ -68,8 +68,23 @@ public class MemoryNameService implements NameService { @Override public List<Record> findRecords(Record.Type type, RecordData data) { + if (type == Record.Type.ALIAS && data.asString().contains("/")) { + // Validate the same expectation as of a real name service + throw new IllegalArgumentException("Finding " + Record.Type.ALIAS + " record by data should only include " + + "the FQDN name"); + } return records.stream() - .filter(record -> record.type() == type && record.data().equals(data)) + .filter(record -> { + if (record.type() == type) { + if (type == Record.Type.ALIAS) { + // Unpack ALIAS record and compare FQDN of data part + return RecordData.fqdn(AliasTarget.from(record.data()).name().value()) + .equals(data); + } + return record.data().equals(data); + } + return false; + }) .collect(Collectors.toUnmodifiableList()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java index a4691001adc..30f5801dabb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -32,6 +32,17 @@ public class NodeHistory { return event; } - public enum Agent { system, application, operator, NodeFailer } + public enum Agent { + operator, + application, + system, + NodeFailer, + Rebalancer, + DirtyExpirer, + FailedExpirer, + InactiveExpirer, + ProvisionedExpirer, + ReservationExpirer + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 78a64b98e2b..d554bba6b9d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -54,6 +54,10 @@ public class NodeRepositoryNode { private String currentOsVersion; @JsonProperty("wantedOsVersion") private String wantedOsVersion; + @JsonProperty("currentFirmwareCheck") + private Long currentFirmwareCheck; + @JsonProperty("wantedFirmwareCheck") + private Long wantedFirmwareCheck; @JsonProperty("failCount") private Integer failCount; @JsonProperty("environment") @@ -76,10 +80,14 @@ public class NodeRepositoryNode { private NodeHistory[] history; @JsonProperty("allowedToBeDown") private Boolean allowedToBeDown; + @JsonProperty("suspendedSinceMillis") + private Long suspendedSinceMillis; @JsonProperty("reports") private Map<String, JsonNode> reports; @JsonProperty("modelName") private String modelName; + @JsonProperty("reservedTo") + private String reservedTo; public String getUrl() { return url; @@ -307,6 +315,14 @@ public class NodeRepositoryNode { return allowedToBeDown; } + public Long suspendedSinceMillis() { + return suspendedSinceMillis; + } + + public void setSuspendedSinceMillis(long suspendedSinceMillis) { + this.suspendedSinceMillis = suspendedSinceMillis; + } + public String getCurrentOsVersion() { return currentOsVersion; } @@ -323,6 +339,22 @@ public class NodeRepositoryNode { this.wantedOsVersion = wantedOsVersion; } + public Long getCurrentFirmwareCheck() { + return currentFirmwareCheck; + } + + public void setCurrentFirmwareCheck(Long currentFirmwareCheck) { + this.currentFirmwareCheck = currentFirmwareCheck; + } + + public Long getWantedFirmwareCheck() { + return wantedFirmwareCheck; + } + + public void setWantedFirmwareCheck(Long wantedFirmwareCheck) { + this.wantedFirmwareCheck = wantedFirmwareCheck; + } + public Map<String, JsonNode> getReports() { return reports; } @@ -339,6 +371,10 @@ public class NodeRepositoryNode { this.modelName = modelName; } + public String getReservedTo() { return reservedTo; } + + public void setReservedTo(String reservedTo) { this.reservedTo = reservedTo; } + @Override public String toString() { return "NodeRepositoryNode{" + @@ -375,6 +411,7 @@ public class NodeRepositoryNode { ", allowedToBeDown=" + allowedToBeDown + ", reports=" + reports + ", modelName=" + modelName + + ", reservedTo=" + reservedTo + '}'; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java index 3b062514e65..5eeb1f7401e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ProvisionResource.java @@ -50,6 +50,11 @@ public interface ProvisionResource { @GET @Path("/node/") + NodeList listNodes(@QueryParam("recursive") boolean recursive, + @QueryParam("hostname") String hostnamesString); + + @GET + @Path("/node/") NodeList listNodesWithParent(@QueryParam("recursive") boolean recursive, @QueryParam("parentHost") String parentHostname); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java index 0237f3d34f4..1d23cd52d23 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java @@ -19,12 +19,15 @@ public class ApplicationSummary { private final ApplicationId application; private final Optional<Instant> lastQueried; private final Optional<Instant> lastWritten; + private final Optional<Instant> lastBuilt; private final Map<ZoneId, Metric> metrics; - public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, Map<ZoneId, Metric> metrics) { + public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, + Optional<Instant> lastBuilt, Map<ZoneId, Metric> metrics) { this.application = Objects.requireNonNull(application); this.lastQueried = Objects.requireNonNull(lastQueried); this.lastWritten = Objects.requireNonNull(lastWritten); + this.lastBuilt = Objects.requireNonNull(lastBuilt); this.metrics = Map.copyOf(Objects.requireNonNull(metrics)); } @@ -40,6 +43,10 @@ public class ApplicationSummary { return lastWritten; } + public Optional<Instant> lastBuilt() { + return lastBuilt; + } + public Map<ZoneId, Metric> metrics() { return metrics; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java index 3e06b24c6be..1047e1a02a4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; +import java.time.YearMonth; import java.util.Collection; +import java.util.List; /** * Consumes and retrieves snapshots of resources allocated per application. @@ -15,6 +17,8 @@ public interface MeteringClient { void consume(Collection<ResourceSnapshot> resources); - MeteringInfo getResourceSnapshots(TenantName tenantName, ApplicationName applicationName); + MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName); + + List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringData.java index d6cb8f7fe76..e9a6c81e636 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringData.java @@ -9,14 +9,14 @@ import java.util.Map; /** * @author olaa */ -public class MeteringInfo { +public class MeteringData { private final ResourceAllocation thisMonth; private final ResourceAllocation lastMonth; private final ResourceAllocation currentSnapshot; Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory; - public MeteringInfo(ResourceAllocation thisMonth, ResourceAllocation lastMonth, ResourceAllocation currentSnapshot, Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory) { + public MeteringData(ResourceAllocation thisMonth, ResourceAllocation lastMonth, ResourceAllocation currentSnapshot, Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory) { this.thisMonth = thisMonth; this.lastMonth = lastMonth; this.currentSnapshot = currentSnapshot; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java index a7dc5f81bc0..cd0e6552596 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java @@ -1,8 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.routing; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.util.List; @@ -27,17 +29,19 @@ public class RoutingGeneratorMock implements RoutingGenerator { private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>(); private final List<RoutingEndpoint> defaultEndpoints; + private final ZoneRegistry zoneRegistry; - public RoutingGeneratorMock() { - this(List.of()); - } - - public RoutingGeneratorMock(List<RoutingEndpoint> endpoints) { + public RoutingGeneratorMock(List<RoutingEndpoint> endpoints, ZoneRegistry zoneRegistry) { this.defaultEndpoints = List.copyOf(endpoints); + this.zoneRegistry = zoneRegistry; } @Override public List<RoutingEndpoint> endpoints(DeploymentId deployment) { + if (!zoneRegistry.zones().routingMethod(RoutingMethod.shared).ids().contains(deployment.zoneId())) { + throw new IllegalArgumentException(deployment.zoneId() + " does not support routing method " + + RoutingMethod.shared); + } if (routingTable.isEmpty()) return defaultEndpoints; return routingTable.getOrDefault(deployment, List.of()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java deleted file mode 100644 index 2bf2a706ee6..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(value = JsonInclude.Include.NON_NULL) -public class StatusReply { - - @JsonProperty("status") public final RotationStatus status; - @JsonProperty("lastUpdate") public final long lastUpdate; - @JsonProperty("cause") public final String cause; - @JsonProperty("agent") public final String agent; - - @JsonCreator - public StatusReply(@JsonProperty("status") RotationStatus status, - @JsonProperty("lastUpdate") long lastUpdate, - @JsonProperty("cause") String cause, - @JsonProperty("agent") String agent) { - this.status = status; - this.lastUpdate = lastUpdate; - this.cause = cause; - this.agent = agent; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java deleted file mode 100644 index aea5f6d928d..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -/** - * @author bjorncs - */ -public enum ZoneStatus { - OUT, IN -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java deleted file mode 100644 index d2e94e73b39..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ZoneStatusReply { - - public final ZoneStatus status; - - @JsonCreator - public ZoneStatusReply(@JsonProperty("status") ZoneStatus status) { - this.status = status; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java deleted file mode 100644 index 0b6117d516f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java index 45ead36f622..80caef1709d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java @@ -3,16 +3,19 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo; +import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import java.time.YearMonth; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * @author olaa @@ -20,7 +23,7 @@ import java.util.Optional; public class MockMeteringClient implements MeteringClient { private Collection<ResourceSnapshot> resources = new ArrayList<>(); - private Optional<MeteringInfo> meteringInfo; + private Optional<MeteringData> meteringData; @Override public void consume(Collection<ResourceSnapshot> resources){ @@ -28,18 +31,25 @@ public class MockMeteringClient implements MeteringClient { } @Override - public MeteringInfo getResourceSnapshots(TenantName tenantName, ApplicationName applicationName) { - return meteringInfo.orElseGet(() -> { + public MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName) { + return meteringData.orElseGet(() -> { ResourceAllocation emptyAllocation = new ResourceAllocation(0, 0, 0); - return new MeteringInfo(emptyAllocation, emptyAllocation, emptyAllocation, Collections.emptyMap()); + return new MeteringData(emptyAllocation, emptyAllocation, emptyAllocation, Collections.emptyMap()); }); } + @Override + public List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth) { + return new ArrayList<>(resources); + } + public Collection<ResourceSnapshot> consumedResources() { return this.resources; } - public void setMeteringInfo(MeteringInfo meteringInfo) { - this.meteringInfo = Optional.of(meteringInfo); + public void setMeteringData(MeteringData meteringData) { + this.meteringData = Optional.of(meteringData); + this.resources = meteringData.getSnapshotHistory().entrySet().stream().map(Map.Entry::getValue).flatMap(List::stream).collect(Collectors.toList()); + boolean a = false; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index ce4d44eadce..d2914f95360 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -1,13 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import java.net.URI; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -29,16 +28,29 @@ public class MockTesterCloud implements TesterCloud { } @Override + public void startTests(DeploymentId deploymentId, Suite suite, byte[] config) { + this.status = RUNNING; + this.config = config; + this.testerUrl = null; + } + + @Override public List<LogEntry> getLog(URI testerUrl, long after) { return log.stream().filter(entry -> entry.id() > after).collect(Collectors.toList()); } @Override - public Status getStatus(URI testerUrl) { - return status; + public List<LogEntry> getLog(DeploymentId deploymentId, long after) { + return log.stream().filter(entry -> entry.id() > after).collect(Collectors.toList()); } @Override + public Status getStatus(URI testerUrl) { return status; } + + @Override + public Status getStatus(DeploymentId deploymentId) { return status; } + + @Override public boolean ready(URI testerUrl) { return true; } @@ -49,10 +61,20 @@ public class MockTesterCloud implements TesterCloud { } @Override + public boolean testerReady(DeploymentId deploymentId) { + return true; + } + + @Override public boolean exists(URI endpointUrl) { return true; } + @Override + public boolean exists(DeploymentId deploymentId) { + return true; + } + public void add(LogEntry entry) { log.add(entry); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java index 77bd589f23b..f5f3ebe8f35 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java @@ -35,6 +35,7 @@ public class Roles { public static Role toRole(String value) { String[] parts = value.split("\\."); if (parts.length == 1 && parts[0].equals("hostedOperator")) return Role.hostedOperator(); + if (parts.length == 1 && parts[0].equals("hostedSupporter")) return Role.hostedSupporter(); if (parts.length == 2) return toRole(TenantName.from(parts[0]), parts[1]); if (parts.length == 3) return toRole(TenantName.from(parts[0]), ApplicationName.from(parts[1]), parts[2]); throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'."); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index bdc1c6e9794..67a6faac606 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -30,7 +30,10 @@ enum PathGroup { "/orchestrator/v1/{*}", "/os/v1/{*}", "/provision/v2/{*}", - "/zone/v2/{*}"), + "/zone/v2/{*}", + "/routing/v1/", + "/routing/v1/status/environment/{*}", + "/routing/v1/inactive/environment/{*}"), /** Paths used for creating and reading user resources. */ user(Optional.of("/api"), @@ -52,7 +55,8 @@ enum PathGroup { Optional.of("/api"), "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", - "/application/v4/tenant/{tenant}/cost/{date}"), + "/application/v4/tenant/{tenant}/cost/{date}", + "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, Optional.of("/api"), @@ -96,7 +100,8 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/service/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/global-rotation/{*}", - "/application/v4/tenant/{tenant}/application/{application}/metering"), + "/application/v4/tenant/{tenant}/application/{application}/metering", + "/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}"), // TODO jonmv: remove /** Path used to restart development nodes. */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index e27fb0fbf27..0e8e3a13f9f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -23,6 +23,11 @@ enum Policy { .on(PathGroup.all()) .in(SystemName.all())), + /** Full access to everything. */ + supporter(Privilege.grant(Action.read) + .on(PathGroup.all()) + .in(SystemName.all())), + /** Full access to user management for a tenant in select systems. */ tenantManager(Privilege.grant(Action.all()) .on(PathGroup.tenantUsers) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index b53cf9162e7..263e3284dbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -28,6 +28,11 @@ public abstract class Role { return new UnboundRole(RoleDefinition.hostedOperator); } + /** Returns a {@link RoleDefinition#hostedSupporter} for the current system. */ + public static UnboundRole hostedSupporter() { + return new UnboundRole(RoleDefinition.hostedSupporter); + } + /** Returns a {@link RoleDefinition#everyone} for the current system. */ public static UnboundRole everyone() { return new UnboundRole(RoleDefinition.everyone); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 58d69512feb..848866f7c33 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -21,6 +21,9 @@ public enum RoleDefinition { /** Deus ex machina. */ hostedOperator(Policy.operator), + /** Machina autem exspiravit. */ + hostedSupporter(Policy.supporter), + /** Base role which every user is part of. */ everyone(Policy.classifiedRead, Policy.classifiedApiRead, diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java index cfb5462e50a..22baedd16b4 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java @@ -27,6 +27,8 @@ public class RolesTest { assertEquals(Role.hostedOperator(), Roles.toRole("hostedOperator")); + assertEquals(Role.hostedSupporter(), + Roles.toRole("hostedSupporter")); assertEquals(Role.tenantOperator(tenant), Roles.toRole("my-tenant.tenantOperator")); assertEquals(Role.applicationReader(tenant, application), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index d153e218640..5348185c276 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.TenantName; import org.junit.Test; import java.net.URI; +import java.util.List; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,6 +31,31 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/prod"))); + assertTrue(mainEnforcer.allows(role, Action.create, URI.create("/routing/v1/inactive/environment/prod/region/us-north-1"))); + } + + @Test + public void supporter_membership() { + Role role = Role.hostedSupporter(); + + // No create update or delete + assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); + assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/controller/v1/foo"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2"))); + assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); + + // But reads is ok (but still only for valid paths) + assertFalse(mainEnforcer.allows(role, Action.read, URI.create("/not/explicitly/defined"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/os/v1/bar"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t1/application/a1"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t2/application/a2"))); + assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); } @Test @@ -133,13 +159,42 @@ public class RoleTest { Action action = Action.update; assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, deployUri)); assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, deployUri)); + assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, deployUri)); assertFalse(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, deployUri)); assertFalse(mainEnforcer.allows(Role.everyone(), action, deployUri)); URI dryrunUri = URI.create("/system-flags/v1/dryrun"); assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, dryrunUri)); assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, dryrunUri)); + assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, dryrunUri)); assertTrue(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, dryrunUri)); assertFalse(mainEnforcer.allows(Role.everyone(), action, dryrunUri)); } + + @Test + public void routing() { + var tenantUrl = URI.create("/routing/v1/status/tenant/t1"); + var applicationUrl = URI.create("/routing/v1/status/tenant/t1/application/a1"); + var instanceUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1"); + var deploymentUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + // Read + for (var url : List.of(tenantUrl, applicationUrl, instanceUrl, deploymentUrl)) { + var allowedRole = Role.reader(TenantName.from("t1")); + var disallowedRole = Role.reader(TenantName.from("t2")); + assertTrue(allowedRole + " can read " + url, mainEnforcer.allows(allowedRole, Action.read, url)); + assertFalse(disallowedRole + " cannot read " + url, mainEnforcer.allows(disallowedRole, Action.read, url)); + } + + // Write + { + var url = URI.create("/routing/v1/inactive/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + var allowedRole = Role.applicationAdmin(TenantName.from("t1"), ApplicationName.from("a1")); + var disallowedRole = Role.applicationAdmin(TenantName.from("t2"), ApplicationName.from("a2")); + assertTrue(allowedRole + " can override status at " + url, mainEnforcer.allows(allowedRole, Action.create, url)); + assertTrue(allowedRole + " can clear status at " + url, mainEnforcer.allows(allowedRole, Action.delete, url)); + assertFalse(disallowedRole + " cannot override status at " + url, mainEnforcer.allows(disallowedRole, Action.create, url)); + assertFalse(disallowedRole + " cannot clear status at " + url, mainEnforcer.allows(disallowedRole, Action.delete, url)); + } + } + } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 0c545020449..158c1b5cd39 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -144,6 +144,13 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>http-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <!-- test --> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index e37d6accd89..9815fbca093 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -107,6 +107,12 @@ public class Application { /** Returns the instances of this application */ public Map<InstanceName, Instance> instances() { return instances; } + /** Returns the instances of this application which are defined in its deployment spec. */ + public Map<InstanceName, Instance> productionInstances() { + return deploymentSpec.instanceNames().stream() + .collect(Collectors.toUnmodifiableMap(Function.identity(), instances::get)); + } + /** Returns the instance with the given name, if it exists. */ public Optional<Instance> get(InstanceName instance) { return Optional.ofNullable(instances.get(instance)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index dfc9574fcd7..17c9e852bd9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,33 +1,31 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -40,17 +38,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -58,31 +49,23 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.endpointcertificates.EndpointCertificateManager; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import java.net.URI; import java.security.Principal; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -92,8 +75,6 @@ import java.util.TreeMap; import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -119,31 +100,29 @@ public class ApplicationController { private final ArtifactRepository artifactRepository; private final ApplicationStore applicationStore; - private final RotationRepository rotationRepository; private final AccessControl accessControl; private final ConfigServer configServer; - private final RoutingGenerator routingGenerator; - private final RoutingPolicies routingPolicies; private final Clock clock; private final DeploymentTrigger deploymentTrigger; private final ApplicationPackageValidator applicationPackageValidator; + private final EndpointCertificateManager endpointCertificateManager; ApplicationController(Controller controller, CuratorDb curator, - AccessControl accessControl, RotationsConfig rotationsConfig, - Clock clock) { + AccessControl accessControl, Clock clock, + SecretStore secretStore, FlagSource flagSource) { + this.controller = controller; this.curator = curator; this.accessControl = accessControl; this.configServer = controller.serviceRegistry().configServer(); - this.routingGenerator = controller.serviceRegistry().routingGenerator(); this.clock = clock; this.artifactRepository = controller.serviceRegistry().artifactRepository(); this.applicationStore = controller.serviceRegistry().applicationStore(); - routingPolicies = new RoutingPolicies(controller); - rotationRepository = new RotationRepository(rotationsConfig, this, curator); deploymentTrigger = new DeploymentTrigger(controller, clock); applicationPackageValidator = new ApplicationPackageValidator(controller); + endpointCertificateManager = new EndpointCertificateManager(controller.zoneRegistry(), curator, secretStore, + controller.serviceRegistry().endpointCertificateProvider(), clock, flagSource); // Update serialization format of all applications Once.after(Duration.ofMinutes(1), () -> { @@ -155,7 +134,7 @@ public class ApplicationController { application = application.with(DeploymentSpec.empty); else for (InstanceName instance : application.get().deploymentSpec().instanceNames()) - if ( ! application.get().instances().keySet().contains(instance)) + if ( ! application.get().instances().containsKey(instance)) application = withNewInstance(application, id.instance(instance)); store(application); }); @@ -227,7 +206,7 @@ public class ApplicationController { public Map<ZoneId, List<String>> contentClustersByZone(Collection<DeploymentId> ids) { Map<ZoneId, List<String>> clusters = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (DeploymentId id : ids) - clusters.put(id.zoneId(), ImmutableList.copyOf(configServer.getContentClusters(id))); + clusters.put(id.zoneId(), List.copyOf(configServer.getContentClusters(id))); return Collections.unmodifiableMap(clusters); } @@ -245,36 +224,6 @@ public class ApplicationController { .orElse(controller.systemVersion()); } - /** Change status of all global endpoints for given deployment */ - public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - var globalEndpoints = findGlobalEndpoints(deployment); - if (globalEndpoints.isEmpty()) throw new IllegalArgumentException(deployment + " has no global endpoints"); - globalEndpoints.forEach(endpoint -> { - try { - configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); - } catch (Exception e) { - throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); - } - }); - } - - /** Get global endpoint status for given deployment */ - public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { - var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>(); - findGlobalEndpoints(deployment).forEach(endpoint -> { - var status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName()); - routingEndpoints.put(endpoint, status); - }); - return Collections.unmodifiableMap(routingEndpoints); - } - - /** Find the global endpoints of given deployment */ - private List<RoutingEndpoint> findGlobalEndpoints(DeploymentId deployment) { - return routingGenerator.endpoints(deployment).stream() - .filter(RoutingEndpoint::isGlobal) - .collect(Collectors.toUnmodifiableList()); - } - /** * Creates a new application for an existing tenant. * @@ -359,7 +308,7 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set<ContainerEndpoint> endpoints; - Optional<ApplicationCertificate> applicationCertificate; + Optional<EndpointCertificateMetadata> endpointCertificateMetadata; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); @@ -395,20 +344,15 @@ public class ApplicationController { validateRun(application.get().require(instance), zone, platformVersion, applicationVersion); } - if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { - // Provisions a new certificate if missing - applicationCertificate = getApplicationCertificate(application.get().require(instance)); - } else { - applicationCertificate = Optional.empty(); - } + endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone); - endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); + endpoints = controller.routingController().registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); ActivateResult result = deploy(instanceId, applicationPackage, zone, options, endpoints, - applicationCertificate.orElse(null)); + endpointCertificateMetadata); lockApplicationOrThrow(applicationId, application -> store(application.with(instanceId.instance(), @@ -458,7 +402,7 @@ public class ApplicationController { for (InstanceName instance : declaredInstances) if (applicationPackage.deploymentSpec().requireInstance(instance).concerns(Environment.prod)) - application = withRotation(applicationPackage.deploymentSpec(), application, instance); + application = controller.routingController().assignRotations(application, instance); store(application); return application; @@ -481,7 +425,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ null); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ Optional.empty()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -489,115 +433,23 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ null); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ Optional.empty()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, Set<ContainerEndpoint> endpoints, - ApplicationCertificate applicationCertificate) { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, endpoints, applicationCertificate, applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, endpoints, endpointCertificateMetadata, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that // any DNS updates can be propagated as early as possible. - routingPolicies.refresh(application, applicationPackage.deploymentSpec(), zone); - } - } - - /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(DeploymentSpec deploymentSpec, LockedApplication application, InstanceName instanceName) { - try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(deploymentSpec, - application.get().require(instanceName), - rotationLock); - application = application.with(instanceName, instance -> instance.with(rotations)); - store(application); // store assigned rotation even if deployment fails + controller.routingController().policies().refresh(application, applicationPackage.deploymentSpec(), zone); } - return application; - } - - /** - * Register endpoints for rotations assigned to given application and zone in DNS. - * - * @return the registered endpoints - */ - private Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { - var containerEndpoints = new HashSet<ContainerEndpoint>(); - boolean registerLegacyNames = deploymentSpec.instance(instance.name()) - .flatMap(DeploymentInstanceSpec::globalServiceId) - .isPresent(); - for (var assignedRotation : instance.rotations()) { - var names = new ArrayList<String>(); - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) - .scope(Endpoint.Scope.global); - - // Skip rotations which do not apply to this zone. Legacy names always point to all zones - if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { - continue; - } - - // Omit legacy DNS names when assigning rotations using <endpoints/> syntax - if (!registerLegacyNames) { - endpoints = endpoints.legacy(false); - } - - // Register names in DNS - var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); - if (rotation.isPresent()) { - endpoints.asList().forEach(endpoint -> { - controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), - RecordData.fqdn(rotation.get().name()), - Priority.normal); - names.add(endpoint.dnsName()); - }); - } - - // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) - names.add(assignedRotation.rotationId().asString()); - containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names)); - } - return Collections.unmodifiableSet(containerEndpoints); - } - - private Optional<ApplicationCertificate> getApplicationCertificate(Instance instance) { - // Re-use certificate if already provisioned - Optional<ApplicationCertificate> applicationCertificate = curator.readApplicationCertificate(instance.id()); - if(applicationCertificate.isPresent()) - return applicationCertificate; - - ApplicationCertificate newCertificate = controller.serviceRegistry().applicationCertificateProvider().requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id())); - curator.writeApplicationCertificate(instance.id(), newCertificate); - - return Optional.of(newCertificate); - } - - /** Returns all valid DNS names of given application */ - private List<String> dnsNamesOf(ApplicationId applicationId) { - List<String> endpointDnsNames = new ArrayList<>(); - - // We add first an endpoint name based on a hash of the applicationId, - // as the certificate provider requires the first CN to be < 64 characters long. - endpointDnsNames.add(Endpoint.createHashedCn(applicationId, controller.system())); - - var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId()); - var rotationEndpoints = Endpoint.of(applicationId).wildcard(); - - var zoneLocalEndpoints = controller.zoneRegistry().zones().directlyRouted().zones().stream().flatMap(zone -> Stream.of( - Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone.getId()), - Endpoint.of(applicationId).wildcard(zone.getId()) - )); - - Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints) - .map(Endpoint.EndpointBuilder::directRouting) - .map(endpoint -> endpoint.on(Endpoint.Port.tls())) - .map(endpointBuilder -> endpointBuilder.in(controller.system())) - .map(Endpoint::dnsName).forEach(endpointDnsNames::add); - - return Collections.unmodifiableList(endpointDnsNames); } private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { @@ -651,59 +503,6 @@ public class ApplicationController { options.deployCurrentVersion); } - /** Returns the endpoints of the deployment, or empty if the request fails */ - public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) { - if ( ! getInstance(deploymentId.applicationId()) - .map(application -> application.deployments().containsKey(deploymentId.zoneId())) - .orElse(deploymentId.applicationId().instance().isTester())) - throw new NotExistsException("Deployment", deploymentId.toString()); - - try { - return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .iterator()); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e); - return Collections.emptyList(); - } - } - - /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ - public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) { - if ( ! getInstance(id.applicationId()) - .map(application -> application.deployments().containsKey(id.zoneId())) - .orElse(id.applicationId().instance().isTester())) - throw new NotExistsException("Deployment", id.toString()); - - // TODO(jvenstad): Swap to use routingPolicies first, when this is ready. - try { - var endpoints = routingGenerator.clusterEndpoints(id); - if ( ! endpoints.isEmpty()) - return endpoints; - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + id, e); - } - return routingPolicies.get(id).stream() - .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), - policy -> policy.endpointIn(controller.system()).url())); - } - - /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ - public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(Collection<DeploymentId> ids) { - Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value)); - for (DeploymentId id : ids) { - var endpoints = clusterEndpoints(id); - if ( ! endpoints.isEmpty()) { - deployments.put(id.zoneId(), endpoints); - } - } - return Collections.unmodifiableMap(deployments); - } - /** * Deletes the the given application. All known instances of the applications will be deleted. * @@ -725,7 +524,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments); for (Instance instance : application.get().instances().values()) { - removeEndpoints(instance); + controller.routingController().removeEndpointsInDns(instance); application = application.without(instance.name()); } @@ -761,24 +560,13 @@ public class ApplicationController { && application.get().deploymentSpec().instanceNames().contains(instanceId.instance())) throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first"); - removeEndpoints(application.get().require(instanceId.instance())); + controller.routingController().removeEndpointsInDns(application.get().require(instanceId.instance())); curator.writeApplication(application.without(instanceId.instance()).get()); controller.jobController().collectGarbage(); log.info("Deleted " + instanceId); }); } - private void removeEndpoints(Instance instance) { - instance.rotations().forEach(assignedRotation -> { - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); - endpoints.asList().stream() - .map(Endpoint::dnsName) - .forEach(name -> { - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); - }); - }); - } - /** * Replace any previous version of this application by this instance * @@ -854,7 +642,7 @@ public class ApplicationController { } catch (NotFoundException ignored) { // ok; already gone } finally { - routingPolicies.refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); + controller.routingController().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); } return application.with(instanceName, instance -> instance.withoutDeploymentIn(zone)); } @@ -896,15 +684,6 @@ public class ApplicationController { instance.id(), zone, platformVersion, applicationVersion, deployment.version(), deployment.applicationVersion())); } - /** Returns the rotation repository, used for managing global rotation assignments */ - public RotationRepository rotationRepository() { - return rotationRepository; - } - - public RoutingPolicies routingPolicies() { - return routingPolicies; - } - /** * Verifies that the application can be deployed to the tenant, following these rules: * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index d3e21f0d399..3d492bc00d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; @@ -72,6 +73,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource private final NameServiceForwarder nameServiceForwarder; private final MavenRepository mavenRepository; private final Metric metric; + private final RoutingController routingController; /** * Creates a controller @@ -80,14 +82,14 @@ public class Controller extends AbstractComponent implements ApplicationIdSource */ @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, - MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric) { + MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore) { this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry, metric); + mavenRepository, serviceRegistry, metric, secretStore); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry, Metric metric) { + ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -101,11 +103,9 @@ public class Controller extends AbstractComponent implements ApplicationIdSource metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); jobController = new JobController(this); - applicationController = new ApplicationController(this, curator, accessControl, - Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"), - clock - ); + applicationController = new ApplicationController(this, curator, accessControl, clock, secretStore, flagSource); tenantController = new TenantController(this, curator, accessControl); + routingController = new RoutingController(this, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null")); auditLogger = new AuditLogger(curator, clock); // Record the version of this controller @@ -123,6 +123,11 @@ public class Controller extends AbstractComponent implements ApplicationIdSource /** Returns the instance controlling deployment jobs. */ public JobController jobController() { return jobController; } + /** Returns the instance controlling routing */ + public RoutingController routingController() { + return routingController; + } + /** Returns the service registry of this */ public ServiceRegistry serviceRegistry() { return serviceRegistry; @@ -146,13 +151,6 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region); } - // TODO: Model the response properly - public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, - String environment, String region, String serviceName, String restPath) { - return serviceRegistry.configServer().getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, - serviceName, restPath); - } - /** Replace the current version status by a new one */ public void updateVersionStatus(VersionStatus newStatus) { VersionStatus currentStatus = versionStatus(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java new file mode 100644 index 00000000000..74304f2e49d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -0,0 +1,244 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * The routing controller encapsulates state and methods for inspecting and manipulating DNS-level routing of traffic + * in a system. + * + * The one stop shop for all your routing needs! + * + * @author mpolden + */ +public class RoutingController { + + private static final Logger log = Logger.getLogger(RoutingController.class.getName()); + + private final Controller controller; + private final RoutingPolicies routingPolicies; + private final RotationRepository rotationRepository; + private final RoutingGenerator routingGenerator; + + public RoutingController(Controller controller, RotationsConfig rotationsConfig) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.routingPolicies = new RoutingPolicies(controller); + this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), + controller.curator()); + this.routingGenerator = controller.serviceRegistry().routingGenerator(); + } + + public RoutingPolicies policies() { + return routingPolicies; + } + + public RotationRepository rotations() { + return rotationRepository; + } + + /** Returns all legacy endpoint URLs for given deployment, including global, in the shared routing layer */ + public List<URI> legacyEndpointsOf(DeploymentId deployment) { + return routingEndpointsOf(deployment).stream() + .map(RoutingEndpoint::endpoint) + .map(URI::create) + .collect(Collectors.toUnmodifiableList()); + } + + /** Returns legacy zone endpoints for given deployment, in the shared routing layer */ + public Map<ClusterSpec.Id, URI> legacyZoneEndpointsOf(DeploymentId deployment) { + if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { + return Map.of(); + } + try { + return routingGenerator.clusterEndpoints(deployment); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + deployment, e); + return Map.of(); + } + } + + /** + * Returns all non-global endpoint URLs for given deployment, grouped by their cluster ID. If deployment supports + * {@link RoutingMethod#exclusive} endpoints defined through routing polices are returned. + */ + public Map<ClusterSpec.Id, URI> zoneEndpointsOf(DeploymentId deployment) { + if ( ! controller.applications().getInstance(deployment.applicationId()) + .map(application -> application.deployments().containsKey(deployment.zoneId())) + .orElse(deployment.applicationId().instance().isTester())) + throw new NotExistsException("Deployment", deployment.toString()); + + // In exclusively routed zones we create endpoint URLs from routing policies + if (supportsRoutingMethod(RoutingMethod.exclusive, deployment.zoneId())) { + return routingPolicies.get(deployment).values().stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), + policy -> policy.endpointIn(controller.system()) + .url())); + } + return legacyZoneEndpointsOf(deployment); + } + + /** Returns all non-global endpoint URLs for given deployments, grouped by their cluster ID and zone */ + public Map<ZoneId, Map<ClusterSpec.Id, URI>> zoneEndpointsOf(Collection<DeploymentId> deployments) { + var endpoints = new TreeMap<ZoneId, Map<ClusterSpec.Id, URI>>(Comparator.comparing(ZoneId::value)); + for (var deployment : deployments) { + var zoneEndpoints = zoneEndpointsOf(deployment); + if (!zoneEndpoints.isEmpty()) { + endpoints.put(deployment.zoneId(), zoneEndpoints); + } + } + return Collections.unmodifiableMap(endpoints); + } + + /** Change status of all global endpoints for given deployment */ + public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { + var globalEndpoints = legacyGlobalEndpointsOf(deployment); + globalEndpoints.forEach(endpoint -> { + try { + controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); + } catch (Exception e) { + throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); + } + }); + } + + /** Get global endpoint status for given deployment */ + public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { + var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>(); + legacyGlobalEndpointsOf(deployment).forEach(endpoint -> { + var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, endpoint.upstreamName()); + routingEndpoints.put(endpoint, status); + }); + return Collections.unmodifiableMap(routingEndpoints); + } + + /** Find the global endpoints of given deployment */ + private List<RoutingEndpoint> legacyGlobalEndpointsOf(DeploymentId deployment) { + return routingEndpointsOf(deployment).stream() + .filter(RoutingEndpoint::isGlobal) + .collect(Collectors.toUnmodifiableList()); + } + + /** + * Assigns one or more global rotations to given application, if eligible. The given application is implicitly + * stored, ensuring that the assigned rotation(s) are persisted when this returns. + */ + public LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { + try (RotationLock rotationLock = rotationRepository.lock()) { + var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + application.get().require(instanceName), + rotationLock); + application = application.with(instanceName, instance -> instance.with(rotations)); + controller.applications().store(application); // store assigned rotation even if deployment fails + } + return application; + } + + /** + * Register endpoints for rotations assigned to given application and zone in DNS. + * + * @return the registered endpoints + */ + public Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { + var containerEndpoints = new HashSet<ContainerEndpoint>(); + boolean registerLegacyNames = deploymentSpec.instance(instance.name()) + .flatMap(DeploymentInstanceSpec::globalServiceId) + .isPresent(); + for (var assignedRotation : instance.rotations()) { + var names = new ArrayList<String>(); + var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); + + // Skip rotations which do not apply to this zone. Legacy names always point to all zones + if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { + continue; + } + + // Omit legacy DNS names when assigning rotations using <endpoints/> syntax + if (!registerLegacyNames) { + endpoints = endpoints.legacy(false); + } + + // Register names in DNS + var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); + if (rotation.isPresent()) { + endpoints.asList().forEach(endpoint -> { + controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), + RecordData.fqdn(rotation.get().name()), + Priority.normal); + names.add(endpoint.dnsName()); + }); + } + + // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) + names.add(assignedRotation.rotationId().asString()); + containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names)); + } + return Collections.unmodifiableSet(containerEndpoints); + } + + /** Remove endpoints in DNS for all rotations assigned to given instance */ + public void removeEndpointsInDns(Instance instance) { + instance.rotations().forEach(assignedRotation -> { + var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), + Priority.normal); + }); + }); + } + + private List<RoutingEndpoint> routingEndpointsOf(DeploymentId deployment) { + if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { + return List.of(); // No rotations/shared routing layer in this zone. + } + try { + return routingGenerator.endpoints(deployment); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoints for " + deployment, e); + return List.of(); + } + } + + private boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { + return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index a8ab847cda6..083984bd13c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -9,7 +9,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; @@ -79,7 +79,7 @@ public class ApplicationPackage { Optional<Inspector> buildMetaObject = files.get("build-meta.json").map(SlimeUtils::jsonToSlime).map(Slime::get); if (requireFiles && buildMetaObject.isEmpty()) - throw new IllegalArgumentException("Missing required file 'deployment.xml'"); + throw new IllegalArgumentException("Missing required file 'build-meta.json'"); this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString()))); this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 9a6daf026c3..c39255fd7a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -111,7 +111,7 @@ public class Endpoint { return URI.create(scheme + "://" + sanitize(namePart(name, separator)) + systemPart(system, separator) + - sanitize(instancePart(application, zone, separator)) + + sanitize(instancePart(application, separator)) + sanitize(application.application().value()) + separator + sanitize(application.tenant().value()) + @@ -144,7 +144,7 @@ public class Endpoint { return zone.region().value() + "." + zone.environment().value(); } - private static String instancePart(ApplicationId application, ZoneId zone, String separator) { + private static String instancePart(ApplicationId application, String separator) { if (application.instance().isDefault()) return ""; // Skip "default" return application.instance().value() + separator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index de3f76d50cd..5e83d0a71a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; @@ -82,7 +83,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which currently have failing jobs on the given version */ public InstanceList failingOn(Version version) { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().lastCompleted().on(version).isEmpty()); + return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing() + .not().withStatus(outOfCapacity) + .lastCompleted().on(version).isEmpty()); } /** Returns the subset of instances which are not pinned to a certain Vespa version. */ @@ -92,7 +95,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are not currently failing any jobs. */ public InstanceList failing() { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().withStatus(RunStatus.outOfCapacity).isEmpty()); + return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().withStatus(outOfCapacity).isEmpty()); } /** Returns the subset of instances which are currently failing an upgrade. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java deleted file mode 100644 index 80a62d94f2e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * Represents the DNS routing policy for a load balancer. A routing policy is uniquely identified by its owner, cluster - * and zone. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicy { - - private final ApplicationId owner; - private final ClusterSpec.Id cluster; - private final ZoneId zone; - private final HostName canonicalName; - private final Optional<String> dnsZone; - private final Set<EndpointId> endpoints; - private final boolean active; - - /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, - Optional<String> dnsZone, Set<EndpointId> endpoints, boolean active) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); - this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); - this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); - this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); - this.active = active; - } - - /** The application owning this */ - public ApplicationId owner() { - return owner; - } - - /** The zone this applies to */ - public ZoneId zone() { - return zone; - } - - /** The cluster this applies to */ - public ClusterSpec.Id cluster() { - return cluster; - } - - /** The canonical name for this (rhs of a CNAME or ALIAS record) */ - public HostName canonicalName() { - return canonicalName; - } - - /** DNS zone for this, if any */ - public Optional<String> dnsZone() { - return dnsZone; - } - - /** The endpoints of this policy */ - public Set<EndpointId> endpoints() { - return endpoints; - } - - /** Returns whether this is active (the underlying load balancer is in an active state) */ - public boolean active() { - return active; - } - - /** Returns the endpoint of this */ - public Endpoint endpointIn(SystemName system) { - return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); - } - - /** Returns rotation endpoints of this */ - public EndpointList rotationEndpointsIn(SystemName system) { - return EndpointList.of(endpoints.stream().map(endpointId -> endpointOf(owner, endpointId, system))); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RoutingPolicy that = (RoutingPolicy) o; - return owner.equals(that.owner) && - cluster.equals(that.cluster) && - zone.equals(that.zone); - } - - @Override - public int hashCode() { - return Objects.hash(owner, cluster, zone); - } - - @Override - public String toString() { - return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, endpoints, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), - zone.value()); - } - - /** Returns the endpoint of given rotation */ - public static Endpoint endpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { - return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 7ace62ab44d..628d7f48c85 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -215,6 +215,10 @@ public class AthenzFacade implements AccessControl { return hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity); } + public boolean hasHostedSupporterAccess(AthenzIdentity identity) { + return hasAccess("read", service.getDomain().getName() + ":hosted-vespa", identity); + } + public boolean canLaunch(AthenzIdentity principal, AthenzService service) { return hasAccess("launch", service.getDomain().getName() + ":service."+service.getName(), principal); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java new file mode 100644 index 00000000000..d874a8042f2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ConvergenceSummary.java @@ -0,0 +1,138 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import java.util.Objects; + +/** + * Summary of node and service status during a deployment job. + * + * @author jonmv + */ +public class ConvergenceSummary { + + private final long nodes; + private final long down; + private final long upgradingOs; + private final long upgradingFirmware; + private final long needPlatformUpgrade; + private final long upgradingPlatform; + private final long needReboot; + private final long rebooting; + private final long needRestart; + private final long restarting; + private final long services; + private final long needNewConfig; + + public ConvergenceSummary(long nodes, long down, long upgradingOs, long upgradingFirmware, long needPlatformUpgrade, long upgradingPlatform, + long needReboot, long rebooting, long needRestart, long restarting, long services, long needNewConfig) { + this.nodes = nodes; + this.down = down; + this.upgradingOs = upgradingOs; + this.upgradingFirmware = upgradingFirmware; + this.needPlatformUpgrade = needPlatformUpgrade; + this.upgradingPlatform = upgradingPlatform; + this.needReboot = needReboot; + this.rebooting = rebooting; + this.needRestart = needRestart; + this.restarting = restarting; + this.services = services; + this.needNewConfig = needNewConfig; + } + + /** Number of nodes in the application. */ + public long nodes() { + return nodes; + } + + /** Number of nodes allowed to be down. */ + public long down() { + return down; + } + + /** Number of nodes down for OS upgrade. */ + public long upgradingOs() { + return upgradingOs; + } + + /** Number of nodes down for firmware upgrade. */ + public long upgradingFirmware() { + return upgradingFirmware; + } + + /** Number of nodes in need of a platform upgrade. */ + public long needPlatformUpgrade() { + return needPlatformUpgrade; + } + + /** Number of nodes down for platform upgrade. */ + public long upgradingPlatform() { + return upgradingPlatform; + } + + /** Number of nodes in need of a reboot. */ + public long needReboot() { + return needReboot; + } + + /** Number of nodes down for reboot. */ + public long rebooting() { + return rebooting; + } + + /** Number of nodes in need of a restart. */ + public long needRestart() { + return needRestart; + } + + /** Number of nodes down for restart. */ + public long restarting() { + return restarting; + } + + /** Number of services in the application. */ + public long services() { + return services; + } + + /** Number of services with outdated config generation. */ + public long needNewConfig() { + return needNewConfig; + } + + /** Whether the convergence is done. */ + public boolean converged() { + return nodes > 0 + && needPlatformUpgrade == 0 + && needReboot == 0 + && needRestart == 0 + && services > 0 + && needNewConfig == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConvergenceSummary that = (ConvergenceSummary) o; + return nodes == that.nodes && + down == that.down && + upgradingOs == that.upgradingOs && + upgradingFirmware == that.upgradingFirmware && + needPlatformUpgrade == that.needPlatformUpgrade && + upgradingPlatform == that.upgradingPlatform && + needReboot == that.needReboot && + rebooting == that.rebooting && + needRestart == that.needRestart && + restarting == that.restarting && + services == that.services && + needNewConfig == that.needNewConfig; + } + + @Override + public int hashCode() { + return Objects.hash(nodes, down, upgradingOs, upgradingFirmware, needPlatformUpgrade, upgradingPlatform, needReboot, rebooting, needRestart, restarting, services, needNewConfig); + } + +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 3a60c480100..6aebae66bad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.time.Instant; @@ -579,8 +578,12 @@ public class DeploymentStatus { Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion); return job.lastSuccess() .filter(run -> versions.targetsMatch(run.versions())) - .filter(run -> status.instanceJobs(instance).get(prodType).lastCompleted() - .map(last -> ! last.end().get().isAfter(run.start())).orElse(false)) + .filter(run -> ! status.jobs() + .instance(instance) + .type(prodType) + .successOn(versions) + .lastCompleted().endedNoLaterThan(run.start()) + .isEmpty()) .map(run -> run.end().get()); } }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index c14493a0b72..efa21b71936 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -3,15 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Collection; -import java.util.List; -import java.util.Optional; /** * List for filtering deployment status of applications, for inspection and decision making. @@ -48,14 +42,14 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus private static boolean failingUpgradeToVersionSince(JobList jobs, Version version, Instant threshold) { return ! jobs.not().failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .lastCompleted().on(version) .isEmpty(); } private static boolean failingApplicationChangeSince(JobList jobs, Instant threshold) { return ! jobs.failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 9d09394a571..9d666c6f7b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; @@ -12,14 +13,12 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -27,9 +26,12 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -39,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Deployment import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; @@ -52,6 +55,7 @@ import java.security.cert.CertificateExpiredException; import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -63,6 +67,7 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; @@ -71,15 +76,23 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; /** * Runs steps of a deployment job against its provided controller. @@ -94,16 +107,18 @@ import static java.util.logging.Level.WARNING; public class InternalStepRunner implements StepRunner { private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName()); - private static final NodeResources DEFAULT_TESTER_RESOURCES = + + static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); // Must match exactly the advertised resources of an AWS instance type. Also consider that the container // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). - private static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = + static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final Duration endpointTimeout = Duration.ofMinutes(15); + static final Duration endpointCertificateTimeout = Duration.ofMinutes(15); static final Duration testerTimeout = Duration.ofMinutes(30); - static final Duration installationTimeout = Duration.ofMinutes(150); + static final Duration installationTimeout = Duration.ofMinutes(60); static final Duration certificateTimeout = Duration.ofMinutes(300); private final Controller controller; @@ -186,6 +201,9 @@ public class InternalStepRunner implements StepRunner { vespaVersion, false, setTheStage)), + controller.jobController().run(id).get() + .stepInfo(setTheStage ? deployInitialReal : deployReal).get() + .startTime().get(), logger); } @@ -201,12 +219,23 @@ public class InternalStepRunner implements StepRunner { Optional.of(platform), false, false)), + controller.jobController().run(id).get() + .stepInfo(deployTester).get() + .startTime().get(), logger); } - private Optional<RunStatus> deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment, DualLogger logger) { + private Optional<RunStatus> deploy(ApplicationId id, JobType type, Supplier<ActivateResult> deployment, + Instant startTime, DualLogger logger) { try { PrepareResponse prepareResponse = deployment.get().prepareResponse(); + if (prepareResponse.log != null) + logger.logAll(prepareResponse.log.stream() + .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. + Instant.ofEpochMilli(entry.time), + LogEntry.typeOf(LogLevel.parse(entry.level)), + entry.message)) + .collect(toList())); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { List<String> messages = new ArrayList<>(); messages.add("Deploy failed due to non-compatible changes that require re-feed."); @@ -220,10 +249,6 @@ public class InternalStepRunner implements StepRunner { .filter(action -> ! action.allowed) .flatMap(action -> action.messages.stream()) .forEach(messages::add); - messages.add("Details:"); - prepareResponse.log.stream() - .map(entry -> entry.message) - .forEach(messages::add); logger.log(messages); return Optional.of(deploymentFailed); } @@ -238,7 +263,7 @@ public class InternalStepRunner implements StepRunner { .map(Hostname::new) .forEach(hostname -> { controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), Optional.of(hostname)); - logger.log("Restarting services on host " + hostname.id() + "."); + logger.log("Schedule service restart on host " + hostname.id() + "."); }); logger.log("Deployment successful."); if (prepareResponse.message != null) @@ -246,17 +271,25 @@ public class InternalStepRunner implements StepRunner { return Optional.of(running); } catch (ConfigServerException e) { + // Retry certain failures for up to one hour. + Optional<RunStatus> result = startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) + ? Optional.of(deploymentFailed) : Optional.empty(); switch (e.getErrorCode()) { + case CERTIFICATE_NOT_READY: + if (startTime.plus(endpointCertificateTimeout).isBefore(controller.clock().instant())) { + logger.log("Deployment failed to find provisioned endpoint certificate after " + endpointCertificateTimeout); + return Optional.of(RunStatus.endpointCertificateTimeout); + } + return result; case ACTIVATION_CONFLICT: case APPLICATION_LOCK_FAILURE: - case CERTIFICATE_NOT_READY: logger.log("Deployment failed with possibly transient error " + e.getErrorCode() + ", will retry: " + e.getMessage()); - return Optional.empty(); + return result; case LOAD_BALANCER_NOT_READY: case PARENT_HOST_NOT_READY: logger.log(e.getServerMessage()); - return Optional.empty(); + return result; case OUT_OF_CAPACITY: logger.log(e.getServerMessage()); return Optional.of(outOfCapacity); @@ -280,18 +313,36 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installReal(RunId id, boolean setTheStage, DualLogger logger) { Optional<Deployment> deployment = deployment(id.application(), id.type()); - if ( ! deployment.isPresent()) { + if (deployment.isEmpty()) { logger.log(INFO, "Deployment expired before installation was successful."); return Optional.of(installationFailed); } Versions versions = controller.jobController().run(id).get().versions(); Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); - ApplicationVersion application = setTheStage ? versions.sourceApplication().orElse(versions.targetApplication()) : versions.targetApplication(); - logger.log("Checking installation of " + platform + " and " + application.id() + " ..."); - if ( nodesConverged(id.application(), id.type(), platform, logger) - && servicesConverged(id.application(), id.type(), platform, logger)) { + Run run = controller.jobController().run(id).get(); + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone(controller.system())), + Optional.of(platform)); + if (services.isEmpty()) { + logger.log("Config status not currently available -- will retry."); + return Optional.empty(); + } + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + id.application(), + ImmutableSet.of(active, reserved)); + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + nodes.stream().map(node -> node.parentHostname().get()).collect(toList())); + NodeList nodeList = NodeList.of(nodes, parents, services.get()); + boolean firstTick = run.convergenceSummary().isEmpty(); + if (firstTick) { // Run the first time (for each convergence step). + logger.log(nodeList.asList().stream() + .flatMap(node -> nodeDetails(node, true)) + .collect(toList())); + } + ConvergenceSummary summary = nodeList.summary(); + if (summary.converged()) { + controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { if (containersAreUp(id.application(), id.type().zone(controller.system()), logger)) { logger.log("Installation succeeded!"); @@ -304,52 +355,94 @@ public class InternalStepRunner implements StepRunner { } } - if (timedOut(id, deployment.get(), installationTimeout)) { - logger.log(INFO, "Installation failed to complete within " + installationTimeout.toMinutes() + " minutes!"); + String failureReason = null; + + NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(installationTimeout)); + if ( ! suspendedTooLong.isEmpty()) { + failureReason = "Some nodes have been suspended for more than " + installationTimeout.toMinutes() + " minutes:\n" + + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); + } + + if (run.noNodesDownSince() + .map(since -> since.isBefore(controller.clock().instant().minus(installationTimeout))) + .orElse(false)) { + if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0) + failureReason ="No nodes allowed to suspend to progress installation for " + installationTimeout.toMinutes() + " minutes."; + else + failureReason = "Nodes not able to start with new application package."; + } + + Duration timeout = JobRunner.jobTimeout.minusHours(1); // Time out before job dies. + if (timedOut(id, deployment.get(), timeout)) { + failureReason = "Installation failed to complete within " + timeout.toHours() + "hours!"; + } + + if (failureReason != null) { + logger.log(nodeList.asList().stream() + .flatMap(node -> nodeDetails(node, true)) + .collect(toList())); + logger.log(INFO, failureReason); return Optional.of(installationFailed); } - logger.log("Installation not yet complete."); + if ( ! firstTick) + logger.log(nodeList.expectedDown().asList().stream() + .flatMap(node -> nodeDetails(node, false)) + .collect(toList())); + + controller.jobController().locked(id, lockedRun -> { + Instant noNodesDownSince = nodeList.allowedDown().size() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null; + return lockedRun.noNodesDownSince(noNodesDownSince).withSummary(summary); + }); + return Optional.empty(); } private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); Version platform = controller.systemVersion(); - logger.log("Checking installation of tester container ..."); - if ( nodesConverged(id.tester().id(), id.type(), platform, logger) - && servicesConverged(id.tester().id(), id.type(), platform, logger)) { - if (endpointsAvailable(id.tester().id(), id.type().zone(controller.system()), logger)) { - if (containersAreUp(id.tester().id(), id.type().zone(controller.system()), logger)) { - logger.log("Tester container successfully installed!"); - return Optional.of(running); - } - } - else if (run.stepInfo(installTester).get().startTime().get().plus(endpointTimeout).isBefore(controller.clock().instant())) { - logger.log(WARNING, "Tester failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); - return Optional.of(error); - } + ZoneId zone = id.type().zone(controller.system()); + ApplicationId testerId = id.tester().id(); + + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), + Optional.of(platform)); + if (services.isEmpty()) { + logger.log("Config status not currently available -- will retry."); + return run.stepInfo(installTester).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(5))) + ? Optional.of(error) + : Optional.empty(); + } + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(zone, + testerId, + ImmutableSet.of(active, reserved)); + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(zone, + nodes.stream().map(node -> node.parentHostname().get()).collect(toList())); + NodeList nodeList = NodeList.of(nodes, parents, services.get()); + logger.log(nodeList.asList().stream() + .flatMap(node -> nodeDetails(node, false)) + .collect(toList())); + + if (nodeList.summary().converged() && testerContainersAreUp(testerId, zone, logger)) { + logger.log("Tester container successfully installed!"); + return Optional.of(running); } - if (run.stepInfo(installTester).get().startTime().get().plus(endpointTimeout).isBefore(controller.clock().instant())) { + if (run.stepInfo(installTester).get().startTime().get().plus(testerTimeout).isBefore(controller.clock().instant())) { logger.log(WARNING, "Installation of tester failed to complete within " + testerTimeout.toMinutes() + " minutes!"); return Optional.of(error); } - logger.log("Installation of tester not yet complete."); return Optional.empty(); } /** Returns true iff all containers in the deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; for (URI endpoint : endpoints.get(zoneId).values()) { - boolean ready = id.instance().isTester() ? controller.jobController().cloud().testerReady(endpoint) - : controller.jobController().cloud().ready(endpoint); - + boolean ready = controller.jobController().cloud().ready(endpoint); if (!ready) { logger.log("Failed to get 100 consecutive OKs from " + endpoint); return false; @@ -359,9 +452,19 @@ public class InternalStepRunner implements StepRunner { return true; } + /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ + private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { + DeploymentId deploymentId = new DeploymentId(id, zoneId); + if (controller.jobController().cloud().testerReady(deploymentId)) { + return true; + } else { + logger.log("Failed to get 100 consecutive OKs from tester container for " + deploymentId); + return false; + } + } + private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - logger.log("Attempting to find deployment endpoints ..."); - var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zone))); + var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zone))); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; @@ -386,46 +489,37 @@ public class InternalStepRunner implements StepRunner { logger.log(messages); } - private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(type.zone(controller.system()), id, ImmutableSet.of(active, reserved)); - List<String> statuses = nodes.stream() - .map(node -> String.format("%70s: %-16s%-25s%-32s%s", - node.hostname(), - node.serviceState(), - node.wantedVersion() + (node.currentVersion().equals(node.wantedVersion()) ? "" : " <-- " + node.currentVersion()), - node.restartGeneration() >= node.wantedRestartGeneration() ? "" - : "restart pending (" + node.wantedRestartGeneration() + " <-- " + node.restartGeneration() + ")", - node.rebootGeneration() >= node.wantedRebootGeneration() ? "" - : "reboot pending (" + node.wantedRebootGeneration() + " <-- " + node.rebootGeneration() + ")")) - .collect(Collectors.toList()); - logger.log(statuses); - - return nodes.stream().allMatch(node -> node.currentVersion().equals(target) - && node.restartGeneration() >= node.wantedRestartGeneration() - && node.rebootGeneration() >= node.wantedRebootGeneration()); - } - - private boolean servicesConverged(ApplicationId id, JobType type, Version platform, DualLogger logger) { - var convergence = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id, type.zone(controller.system())), - Optional.of(platform)); - if (convergence.isEmpty()) { - logger.log("Config status not currently available -- will retry."); - return false; + private Stream<String> nodeDetails(NodeWithServices node, boolean printAllServices) { + return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()) + (node.node().suspendedSince().map(since -> " since " + since).orElse("")), + "--- platform " + node.node().wantedVersion() + (node.needsPlatformUpgrade() + ? " <-- " + (node.node().currentVersion().isEmpty() ? "not booted" : node.node().currentVersion()) + : "") + + (node.needsOsUpgrade() && node.isAllowedDown() + ? ", upgrading OS (" + node.node().wantedOsVersion() + " <-- " + node.node().currentOsVersion() + ")" + : "") + + (node.needsFirmwareUpgrade() && node.isAllowedDown() + ? ", upgrading firmware" + : "") + + (node.needsRestart() + ? ", restart pending (" + node.node().wantedRestartGeneration() + " <-- " + node.node().restartGeneration() + ")" + : "") + + (node.needsReboot() + ? ", reboot pending (" + node.node().wantedRebootGeneration() + " <-- " + node.node().rebootGeneration() + ")" + : "")), + node.services().stream() + .filter(service -> printAllServices || node.needsNewConfig()) + .map(service -> "--- " + service.type() + " on port " + service.port() + (service.currentGeneration() == -1 + ? " has not started " + : " has config generation " + service.currentGeneration() + ", wanted is " + node.wantedConfigGeneration()))); + } + + private String humanize(Node.ServiceState state) { + switch (state) { + case allowedDown: return "allowed to be DOWN"; + case expectedUp: return "expected to be UP"; + case unorchestrated: return "unorchestrated"; + default: return state.name(); } - logger.log("Wanted config generation is " + convergence.get().wantedGeneration()); - List<String> statuses = convergence.get().services().stream() - .filter(serviceStatus -> serviceStatus.currentGeneration() != convergence.get().wantedGeneration()) - .map(serviceStatus -> String.format("%70s: %11s on port %4d has config generation %s", - serviceStatus.host().value(), - serviceStatus.type(), - serviceStatus.port(), - serviceStatus.currentGeneration() == -1 ? "not started!" : Long.toString(serviceStatus.currentGeneration()))) - .collect(Collectors.toList()); - logger.log(statuses); - if (statuses.isEmpty()) - logger.log("All services on wanted config generation."); - - return convergence.get().converged(); } private Optional<RunStatus> startTests(RunId id, boolean isSetup, DualLogger logger) { @@ -439,35 +533,30 @@ public class InternalStepRunner implements StepRunner { .productionDeployments().keySet().stream() .map(zone -> new DeploymentId(id.application(), zone)) .collect(Collectors.toSet()); - deployments.add(new DeploymentId(id.application(), id.type().zone(controller.system()))); + ZoneId zoneId = id.type().zone(controller.system()); + deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.applications().clusterEndpoints(deployments); - if ( ! endpoints.containsKey(id.type().zone(controller.system()))) { + var endpoints = controller.routingController().zoneEndpointsOf(deployments); + if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); } logEndpoints(endpoints, logger); - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if (testerEndpoint.isEmpty()) { - logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!"); - return Optional.of(error); - } - - if ( ! controller.jobController().cloud().testerReady(testerEndpoint.get())) { + if (!controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) { logger.log(WARNING, "Tester container went bad!"); return Optional.of(error); } logger.log("Starting tests ..."); - controller.jobController().cloud().startTests(testerEndpoint.get(), - TesterCloud.Suite.of(id.type(), isSetup), - testConfigSerializer.configJson(id.application(), - id.type(), - true, - endpoints, - controller.applications().contentClustersByZone(deployments))); + TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup); + byte[] config = testConfigSerializer.configJson(id.application(), + id.type(), + true, + endpoints, + controller.applications().contentClustersByZone(deployments)); + controller.jobController().cloud().startTests(getTesterDeploymentId(id), suite, config); return Optional.of(running); } @@ -490,20 +579,7 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestLog(id); - BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); - ZoneId zoneId = id.type().zone(controller.system()); - TesterCloud.Status testStatus; - if (useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value()) { - testStatus = controller.serviceRegistry().configServer().getTesterStatus(new DeploymentId(id.application(), zoneId)); - } else { - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if (testerEndpoint.isEmpty()) { - logger.log("Endpoints for tester not found -- trying again later."); - return Optional.empty(); - } - testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); - } - + TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(getTesterDeploymentId(id)); switch (testStatus) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); @@ -537,48 +613,34 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - return retrying(10, () -> { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return running; - }); + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + return Optional.of(running); } catch (RuntimeException e) { logger.log(WARNING, "Failed deleting application " + id.application(), e); - return Optional.of(error); + Instant startTime = controller.jobController().run(id).get().stepInfo(deactivateReal).get().startTime().get(); + return startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) + ? Optional.of(error) + : Optional.empty(); } } private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - return retrying(10, () -> { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return running; - }); + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().deactivateTester(id.tester(), id.type()); + return Optional.of(running); } catch (RuntimeException e) { logger.log(WARNING, "Failed deleting tester of " + id.application(), e); - return Optional.of(error); + Instant startTime = controller.jobController().run(id).get().stepInfo(deactivateTester).get().startTime().get(); + return startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1))) + ? Optional.of(error) + : Optional.empty(); } } - private static Optional<RunStatus> retrying(int retries, Supplier<RunStatus> task) { - RuntimeException exception = null; - do { - try { - return Optional.of(task.get()); - } - catch (RuntimeException e) { - if (exception == null) - exception = e; - else - exception.addSuppressed(e); - } - } while (--retries >= 0); - throw exception; - } - private Optional<RunStatus> report(RunId id, DualLogger logger) { try { controller.jobController().active(id).ifPresent(run -> { @@ -648,7 +710,7 @@ public class InternalStepRunner implements StepRunner { // TODO jonmv: This is a workaround for new deployment writes not yet being visible in spite of Curator locking. // TODO Investigate what's going on here, and remove this workaround. Run run = controller.jobController().run(id).get(); - if (run.start().isAfter(deployment.at())) + if ( ! controller.system().isCd() && run.start().isAfter(deployment.at())) return false; Duration timeout = controller.zoneRegistry().getDeploymentTimeToLive(deployment.zone()) @@ -665,13 +727,9 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(controller.system()); boolean useTesterCertificate = controller.system().isPublic() && id.type().environment().isTest(); - byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(), - ! controller.system().isPublic(), + byte[] servicesXml = servicesXml(! controller.system().isPublic(), useTesterCertificate, - testerFlavorFor(id, spec) - .map(NodeResources::fromLegacyName) - .orElse(zone.region().value().contains("aws-") ? - DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES)); + testerResourcesFor(zone, spec.requireInstance(id.application().instance()))); byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version); byte[] deploymentXml = deploymentXml(id.tester(), spec.athenzDomain(), @@ -704,17 +762,23 @@ public class InternalStepRunner implements StepRunner { zipBuilder.add("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8)); } - private static Optional<String> testerFlavorFor(RunId id, DeploymentSpec spec) { - for (DeploymentSpec.Step step : spec.steps()) - if (step.concerns(id.type().environment())) - return step.zones().get(0).testerFlavor(); + private DeploymentId getTesterDeploymentId(RunId runId) { + ZoneId zoneId = runId.type().zone(controller.system()); + return new DeploymentId(runId.tester().id(), zoneId); + } - return Optional.empty(); + static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + return spec.steps().stream() + .filter(step -> step.concerns(zone.environment())) + .findFirst() + .flatMap(step -> step.zones().get(0).testerFlavor()) + .map(NodeResources::fromLegacyName) + .orElse(zone.region().value().contains("aws-") ? + DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES); } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(AthenzDomain domain, boolean systemUsesAthenz, boolean useTesterCertificate, - NodeResources resources) { + static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, NodeResources resources) { int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); @@ -725,7 +789,6 @@ public class InternalStepRunner implements StepRunner { "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); - AthenzDomain idDomain = ("vespa.vespa.cd".equals(domain.value()) ? AthenzDomain.from("vespa.vespa") : domain); String servicesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<services xmlns:deploy='vespa' version='1.0'>\n" + @@ -744,51 +807,6 @@ public class InternalStepRunner implements StepRunner { " <binding>http://*/tester/v1/*</binding>\n" + " </handler>\n" + "\n" + - " <http>\n" + - " <!-- Make sure 4080 is the first port. This will be used by the config server. -->\n" + - " <server id='default' port='4080'/>\n" + - " <server id='testertls4443' port='4443'>\n" + - " <config name=\"jdisc.http.connector\">\n" + - " <tlsClientAuthEnforcer>\n" + - " <enable>true</enable>\n" + - " <pathWhitelist>\n" + - " <item>/status.html</item>\n" + - " <item>/state/v1/config</item>\n" + - " </pathWhitelist>\n" + - " </tlsClientAuthEnforcer>\n" + - " </config>\n" + - " <ssl>\n" + - " <private-key-file>/var/lib/sia/keys/" + idDomain.value() + ".tenant.key.pem</private-key-file>\n" + - " <certificate-file>/var/lib/sia/certs/" + idDomain.value() + ".tenant.cert.pem</certificate-file>\n" + - " <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file>\n" + - " <client-authentication>want</client-authentication>\n" + - " </ssl>\n" + - " </server>\n" + - " <filtering>\n" + - (systemUsesAthenz ? - " <access-control domain='" + domain.value() + "'>\n" + // Set up dummy access control to pass validation :/ - " <exclude>\n" + - " <binding>http://*/tester/v1/*</binding>\n" + - " </exclude>\n" + - " </access-control>\n" - : "") + - " <request-chain id=\"testrunner-api\">\n" + - " <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle=\"jdisc-security-filters\">\n" + - " <config name=\"jdisc.http.filter.security.athenz.athenz-authorization-filter\">\n" + - " <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>\n" + - " <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>\n" + - " </config>\n" + - " <component id=\"com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper\" bundle=\"jdisc-security-filters\">\n" + - " <config name=\"jdisc.http.filter.security.athenz.static-request-resource-mapper\">\n" + - " <resourceName>" + domain.value() + ":tester-application</resourceName>\n" + - " <action>deploy</action>\n" + - " </config>\n" + - " </component>\n" + - " </filter>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n" + - "\n" + " <nodes count=\"1\" allocated-memory=\"" + jdiscMemoryPct + "%\">\n" + " " + resourceString + "\n" + " </nodes>\n" + @@ -825,6 +843,10 @@ public class InternalStepRunner implements StepRunner { log(List.of(messages)); } + private void logAll(List<LogEntry> messages) { + controller.jobController().log(id, step, messages); + } + private void log(List<String> messages) { controller.jobController().log(id, step, INFO, messages); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b23d16767be..be189004f6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,10 +1,11 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -72,8 +73,8 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class JobController { - private static final int historyLength = 64; - private static final Duration maxHistoryAge = Duration.ofDays(60); + public static final int historyLength = 64; + public static final Duration maxHistoryAge = Duration.ofDays(60); private final Controller controller; private final CuratorDb curator; @@ -179,11 +180,8 @@ public class JobController { if (step.isEmpty()) return run; - Optional<URI> testerEndpoint = testerEndpoint(id); - if ( ! testerEndpoint.isPresent()) - return run; - - List<LogEntry> entries = cloud.getLog(testerEndpoint.get(), run.lastTestLogEntry()); + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -226,9 +224,14 @@ public class JobController { /** Returns an immutable map of all known runs for the given application and job type. */ public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) { - NavigableMap<RunId, Run> runs = curator.readHistoricRuns(id, type); - last(id, type).ifPresent(run -> runs.put(run.id(), run)); - return Collections.unmodifiableNavigableMap(runs); + ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number)); + Optional<Run> last = last(id, type); + curator.readHistoricRuns(id, type).forEach((runId, run) -> { + if (last.isEmpty() || ! runId.equals(last.get().id())) + runs.put(runId, run); + }); + last.ifPresent(run -> runs.put(run.id(), run)); + return runs.build(); } /** Returns the run with the given id, if it exists. */ @@ -353,7 +356,9 @@ public class JobController { old = oldEntries.next()) { // Make sure we keep the last success and the first failing - if (successes == 1 && old.getValue().status() == RunStatus.success) { + if ( successes == 1 + && old.getValue().status() == RunStatus.success + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } @@ -493,12 +498,17 @@ public class JobController { }); } + // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate public void deactivateTester(TesterId id, JobType type) { + var zone = type.zone(controller.system()); try { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); - } - catch (NotFoundException ignored) { + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone)); + } catch (NotFoundException ignored) { // Already gone -- great! + } finally { + // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and + // tester instances have none. + controller.routingController().policies().refresh(id.id(), DeploymentSpec.empty, zone); } } @@ -526,14 +536,10 @@ public class JobController { .collect(toList())); } - /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ + /** Returns the tester endpoint URL, if any */ Optional<URI> testerEndpoint(RunId id) { - DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); - return controller.applications().getDeploymentEndpoints(testerId) - .stream().findAny() - .or(() -> controller.applications().routingPolicies().get(testerId).stream() - .findAny() - .map(policy -> policy.endpointIn(controller.system()).url())); + var testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); + return controller.routingController().zoneEndpointsOf(testerId).values().stream().findFirst(); } private void prunePackages(TenantAndApplicationId id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index f353910163f..525eadb6eaf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -117,7 +117,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return new RunFilter(JobStatus::firstFailing); } - /** Allows sub-filters for runs of the given kind */ + /** Allows sub-filters for runs of the indicated kind */ public class RunFilter { private final Function<JobStatus, Optional<Run>> which; @@ -126,47 +126,32 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { this.which = which; } - /** Returns the subset of jobs where the run of the given type exists */ + /** Returns the subset of jobs where the run of the indicated type exists */ public JobList present() { return matching(run -> true); } - /** Returns the runs of the given kind, mapped by the given function, as a list. */ + /** Returns the runs of the indicated kind, mapped by the given function, as a list. */ public <OtherType> List<OtherType> mapToList(Function<? super Run, OtherType> mapper) { return present().mapToList(which.andThen(Optional::get).andThen(mapper)); } - /** Returns the runs of the given kind. */ + /** Returns the runs of the indicated kind. */ public List<Run> asList() { return mapToList(Function.identity()); } - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList endedBefore(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold)); + /** Returns the subset of jobs where the run of the indicated type ended no later than the given instant */ + public JobList endedNoLaterThan(Instant threshold) { + return matching(run -> ! run.end().orElse(Instant.MAX).isAfter(threshold)); } - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList endedAfter(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MIN).isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList startedBefore(Instant threshold) { - return matching(run -> run.start().isBefore(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList startedAfter(Instant threshold) { - return matching(run -> run.start().isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(ApplicationVersion version) { return matching(run -> run.versions().targetApplication().equals(version)); } - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(Version version) { return matching(run -> run.versions().targetPlatform().equals(version)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index a6ffb56492f..80924c3c0aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -16,6 +16,7 @@ public class JobMetrics { public static final String start = "deployment.start"; public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String endpointCertificateTimeout = "deployment.endpointCertificateTimeout"; public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; public static final String testFailure = "deployment.testFailure"; @@ -40,17 +41,17 @@ public class JobMetrics { } Map<String, String> contextOf(JobId id) { - return Map.of("tenant", id.application().tenant().value(), - "application", id.application().application().value(), - "instance", id.application().instance().value(), - "job", id.type().jobName(), - "environment", id.type().environment().value(), - "region", id.type().zone(system).region().value()); + return Map.of("applicationId", id.application().toFullString(), + "tenantName", id.application().tenant().value(), + "app", id.application().application().value() + "." + id.application().instance().value(), + "test", Boolean.toString(id.type().isTest()), + "zone", id.type().zone(system).value()); } static String valueOf(RunStatus status) { switch (status) { case outOfCapacity: return outOfCapacity; + case endpointCertificateTimeout: return endpointCertificateTimeout; case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; case testFailure: return testFailure; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 4c2b53d6ba2..88b3442bd6e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -56,13 +56,8 @@ public enum JobProfile { report)), production(EnumSet.of(deployReal, - installReal, - deployTester, - installTester, - startTests, - endTests), - EnumSet.of(deactivateTester, - report)), + installReal), + EnumSet.of(report)), productionTest(EnumSet.of(deployTester, installTester, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java new file mode 100644 index 00000000000..fbfdac427e4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java @@ -0,0 +1,106 @@ +package com.yahoo.vespa.hosted.controller.deployment; + + +import com.yahoo.collections.AbstractFilteringList; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.groupingBy; + +public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList> { + + private final long wantedConfigGeneration; + + private NodeList(Collection<? extends NodeWithServices> items, boolean negate, long wantedConfigGeneration) { + super(items, negate, (i, n) -> new NodeList(i, n, wantedConfigGeneration)); + this.wantedConfigGeneration = wantedConfigGeneration; + } + + public static NodeList of(List<Node> nodes, List<Node> parents, ServiceConvergence services) { + var servicesByHostName = services.services().stream() + .collect(groupingBy(service -> service.host())); + var parentsByHostName = parents.stream() + .collect(Collectors.toMap(node -> node.hostname(), node -> node)); + return new NodeList(nodes.stream() + .map(node -> new NodeWithServices(node, + parentsByHostName.get(node.parentHostname().get()), + services.wantedGeneration(), + servicesByHostName.getOrDefault(node.hostname(), List.of()))) + .collect(Collectors.toList()), + false, + services.wantedGeneration()); + } + + /** The nodes on an outdated OS. */ + public NodeList needsOsUpgrade() { + return matching(NodeWithServices::needsOsUpgrade); + } + + /** The nodes with outdated firmware. */ + public NodeList needsFirmwareUpgrade() { + return matching(NodeWithServices::needsFirmwareUpgrade); + } + + /** The nodes whose parent is down. */ + public NodeList withParentDown() { + return matching(NodeWithServices::hasParentDown); + } + + /** The nodes on an outdated platform. */ + public NodeList needsPlatformUpgrade() { + return matching(NodeWithServices::needsPlatformUpgrade); + } + + /** The nodes in need of a reboot. */ + public NodeList needsReboot() { + return matching(NodeWithServices::needsReboot); + } + + /** The nodes in need of a restart. */ + public NodeList needsRestart() { + return matching(NodeWithServices::needsRestart); + } + + /** The nodes currently allowed to be down. */ + public NodeList allowedDown() { + return matching(node -> node.isAllowedDown()); + } + + /** The nodes currently expected to be down. */ + public NodeList expectedDown() { + return matching(node -> node.isAllowedDown() || node.isNewlyProvisioned()); + } + + /** The nodes which have been suspended since before the given instant. */ + public NodeList suspendedSince(Instant instant) { + return matching(node -> node.isSuspendedSince(instant)); + } + + /** The nodes with services on outdated config generation. */ + public NodeList needsNewConfig() { + return matching(NodeWithServices::needsNewConfig); + } + + /** Returns a summary of the convergence status of the nodes in this list. */ + public ConvergenceSummary summary() { + NodeList allowedDown = expectedDown(); + return new ConvergenceSummary(size(), + allowedDown.size(), + withParentDown().needsOsUpgrade().size(), + withParentDown().needsFirmwareUpgrade().size(), + needsPlatformUpgrade().size(), + allowedDown.needsPlatformUpgrade().size(), + needsReboot().size(), + allowedDown.needsReboot().size(), + needsRestart().size(), + allowedDown.needsRestart().size(), + asList().stream().mapToLong(node -> node.services().size()).sum(), + asList().stream().mapToLong(node -> node.services().stream().filter(service -> wantedConfigGeneration > service.currentGeneration()).count()).sum()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java new file mode 100644 index 00000000000..80c1fe0f40b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java @@ -0,0 +1,82 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; + +import java.time.Instant; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Aggregate of a node and its services, fetched from different sources. + * + * @author jonmv + */ +public class NodeWithServices { + + private final Node node; + private final Node parent; + private final long wantedConfigGeneration; + private final List<ServiceConvergence.Status> services; + + public NodeWithServices(Node node, Node parent, long wantedConfigGeneration, List<ServiceConvergence.Status> services) { + this.node = requireNonNull(node); + this.parent = requireNonNull(parent); + if (wantedConfigGeneration <= 0) + throw new IllegalArgumentException("Wanted config generation must be positive"); + this.wantedConfigGeneration = wantedConfigGeneration; + this.services = List.copyOf(services); + } + + public Node node() { return node; } + public Node parent() { return parent; } + public long wantedConfigGeneration() { return wantedConfigGeneration; } + public List<ServiceConvergence.Status> services() { return services; } + + public boolean needsOsUpgrade() { + return parent.wantedOsVersion().isAfter(parent.currentOsVersion()); + } + + public boolean needsFirmwareUpgrade(){ + return parent.wantedFirmwareCheck() + .map(wanted -> parent.currentFirmwareCheck() + .map(wanted::isAfter) + .orElse(true)) + .orElse(false); + } + + public boolean hasParentDown() { + return parent.serviceState() == Node.ServiceState.allowedDown; + } + + public boolean needsPlatformUpgrade() { + return node.wantedVersion().isAfter(node.currentVersion()); + } + + public boolean needsReboot() { + return node.wantedRebootGeneration() > node.rebootGeneration(); + } + + public boolean needsRestart() { + return node.wantedRestartGeneration() > node.restartGeneration(); + } + + public boolean isAllowedDown() { + return node.serviceState() == Node.ServiceState.allowedDown; + } + + public boolean isNewlyProvisioned() { + return node.currentVersion().equals(Version.emptyVersion); + } + + public boolean isSuspendedSince(Instant instant) { + return node.suspendedSince().map(instant::isAfter).orElse(false); + } + + public boolean needsNewConfig() { + return services.stream().anyMatch(service -> wantedConfigGeneration > service.currentGeneration()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 00021c4765a..8cd57fa7d3a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -35,12 +35,14 @@ public class Run { private final RunStatus status; private final long lastTestRecord; private final Instant lastVespaLogTimestamp; + private final Optional<Instant> noNodesDownSince; + private final Optional<ConvergenceSummary> convergenceSummary; private final Optional<X509Certificate> testerCertificate; // For deserialisation only -- do not use! - public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, - Optional<Instant> end, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, - Optional<X509Certificate> testerCertificate) { + public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end, + RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, + Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -49,6 +51,8 @@ public class Run { this.status = status; this.lastTestRecord = lastTestRecord; this.lastVespaLogTimestamp = lastVespaLogTimestamp; + this.noNodesDownSince = noNodesDownSince; + this.convergenceSummary = convergenceSummary; this.testerCertificate = testerCertificate; } @@ -56,7 +60,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, - -1, Instant.EPOCH, Optional.empty()); + -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); } /** Returns a new Run with the status of the given completed step set accordingly. */ @@ -70,7 +74,7 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(Step.Status.of(status))); return new Run(id, steps, versions, start, end, this.status == running ? status : this.status, - lastTestRecord, lastVespaLogTimestamp, testerCertificate); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate); } /** Returns a new Run with a new start time*/ @@ -84,37 +88,50 @@ public class Run { EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps); steps.put(step.get(), stepInfo.with(startTime)); - return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, testerCertificate); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate); } public Run finished(Instant now) { requireActive(); return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status, - lastTestRecord, lastVespaLogTimestamp, testerCertificate); + lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty()); } public Run aborted() { requireActive(); - return new Run(id, steps, versions, start, end, aborted, - lastTestRecord, lastVespaLogTimestamp, testerCertificate); + return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(long lastTestRecord) { requireActive(); - return new Run(id, steps, versions, start, end, status, - lastTestRecord, lastVespaLogTimestamp, testerCertificate); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate); } public Run with(Instant lastVespaLogTimestamp) { requireActive(); - return new Run(id, steps, versions, start, end, status, - lastTestRecord, lastVespaLogTimestamp, testerCertificate); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, testerCertificate); + } + + public Run noNodesDownSince(Instant noNodesDownSince) { + requireActive(); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate); + } + + public Run withSummary(ConvergenceSummary convergenceSummary) { + requireActive(); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate); } public Run with(X509Certificate testerCertificate) { requireActive(); - return new Run(id, steps, versions, start, end, status, - lastTestRecord, lastVespaLogTimestamp, Optional.of(testerCertificate)); + return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp, + noNodesDownSince, convergenceSummary, Optional.of(testerCertificate)); } /** Returns the id of this run. */ @@ -127,7 +144,7 @@ public class Run { return steps.containsKey(step); } - /** Returns info on step. */ + /** Returns info on step, or empty if the given step is not a part of this run. */ public Optional<StepInfo> stepInfo(Step step) { return Optional.ofNullable(steps.get(step)); } @@ -136,7 +153,7 @@ public class Run { return stepInfo(step).orElseThrow(() -> new IllegalArgumentException("There is no such step " + step + " for run " + id)); } - /** Returns status of step. */ + /** Returns status of step, or empty if the given step is not a part of this run. */ public Optional<Step.Status> stepStatus(Step step) { return stepInfo(step).map(StepInfo::status); } @@ -190,6 +207,16 @@ public class Run { return lastVespaLogTimestamp; } + /** Returns the timestamp of the last time no nodes were allowed to be down. */ + public Optional<Instant> noNodesDownSince() { + return noNodesDownSince; + } + + /** Returns a summary of convergence status during an application deployment — staging or upgrade. */ + public Optional<ConvergenceSummary> convergenceSummary() { + return convergenceSummary; + } + /** Returns the tester certificate for this run, or empty. */ public Optional<X509Certificate> testerCertificate() { return testerCertificate; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 4d0b7ef3b90..fba3f7ae6e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -17,6 +17,9 @@ public enum RunStatus { /** Deployment of the real application was rejected. */ deploymentFailed, + /** Deployment timed out waiting for endpoint certificate */ + endpointCertificateTimeout, + /** Installation of the real application timed out. */ installationFailed, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index a083e2d0210..0dfe945af2b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.io.IOException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java new file mode 100644 index 00000000000..d915da21603 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -0,0 +1,254 @@ +package com.yahoo.vespa.hosted.controller.endpointcertificates; + +import com.google.common.collect.Sets; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.log.LogLevel; +import com.yahoo.security.SubjectAlternativeName; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Looks up stored endpoint certificate metadata, provisions new certificates if none is found, + * and refreshes certificates if a newer version is available. + * + * @author andreer + */ +public class EndpointCertificateManager { + + private static final Logger log = Logger.getLogger(EndpointCertificateManager.class.getName()); + + private final ZoneRegistry zoneRegistry; + private final CuratorDb curator; + private final SecretStore secretStore; + private final EndpointCertificateProvider endpointCertificateProvider; + private final Clock clock; + private final BooleanFlag useRefreshedEndpointCertificate; + private final StringFlag endpointCertificateBackfill; + private final BooleanFlag endpointCertInSharedRouting; + + public EndpointCertificateManager(ZoneRegistry zoneRegistry, + CuratorDb curator, + SecretStore secretStore, + EndpointCertificateProvider endpointCertificateProvider, + Clock clock, FlagSource flagSource) { + this.zoneRegistry = zoneRegistry; + this.curator = curator; + this.secretStore = secretStore; + this.endpointCertificateProvider = endpointCertificateProvider; + this.clock = clock; + this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource); + this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource); + this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { + try { + this.backfillCertificateMetadata(); + } catch (Throwable t) { + log.log(LogLevel.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t); + } + }, 1, 10, TimeUnit.MINUTES); + } + + public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { + + boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); + if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) return Optional.empty(); + + // Re-use existing certificate if already provisioned + var endpointCertificateMetadata = + curator.readEndpointCertificateMetadata(instance.id()) + .orElseGet(() -> provisionEndpointCertificate(instance)); + + // If feature flag set for application, look for and use refreshed certificate + if (useRefreshedEndpointCertificate.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value()) { + var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata); + + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) { + var refreshedCertificateMetadata = new EndpointCertificateMetadata( + endpointCertificateMetadata.keyName(), + endpointCertificateMetadata.certName(), + latestAvailableVersion.getAsInt() + ); + + if (verifyEndpointCertificate(refreshedCertificateMetadata, instance, zone, "Did not refresh, problems with refreshed certificate: ")) + return Optional.of(refreshedCertificateMetadata); + } + } + + // Only log warnings + verifyEndpointCertificate(endpointCertificateMetadata, instance, zone, "Problems while verifying certificate: "); + + return Optional.of(endpointCertificateMetadata); + } + + enum BackfillMode { + DISABLE, + DRYRUN, + ENABLE + } + + private void backfillCertificateMetadata() { + BackfillMode mode = BackfillMode.valueOf(endpointCertificateBackfill.value()); + if (mode == BackfillMode.DISABLE) return; + + List<EndpointCertificateMetadata> allProviderCertificateMetadata = endpointCertificateProvider.listCertificates(); + Map<String, EndpointCertificateMetadata> sanToEndpointCertificate = new HashMap<>(); + + allProviderCertificateMetadata.forEach((providerMetadata -> { + if (providerMetadata.request_id().isEmpty()) + throw new RuntimeException("Backfill failed - provider metadata missing request_id"); + if (providerMetadata.requestedDnsSans().isEmpty()) + throw new RuntimeException("Backfill failed - provider metadata missing DNS SANs for " + providerMetadata.request_id().get()); + providerMetadata.requestedDnsSans().get().forEach(san -> sanToEndpointCertificate.put(san, providerMetadata) + ); + })); + + Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = curator.readAllEndpointCertificateMetadata(); + + allEndpointCertificateMetadata.forEach((applicationId, storedMetaData) -> { + if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent()) + return; + + var hashedCn = Endpoint.createHashedCn(applicationId, zoneRegistry.system()); // use as join key + EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn); + + if(providerMetadata == null) { + log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); + return; + } + + EndpointCertificateMetadata backfilledMetadata = + new EndpointCertificateMetadata( + storedMetaData.keyName(), + storedMetaData.certName(), + storedMetaData.version(), + providerMetadata.request_id(), + providerMetadata.requestedDnsSans()); + + if (mode == BackfillMode.DRYRUN) { + log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); + } else if (mode == BackfillMode.ENABLE) { + curator.writeEndpointCertificateMetadata(applicationId, backfilledMetadata); + } + }); + } + + private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { + var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); + var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); + + return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); + } + + private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { + List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); + EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider + .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones)); + curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); + return provisionedCertificateMetadata; + } + + private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone, String warningPrefix) { + try { + var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); + + if (pemEncodedEndpointCertificate == null) + return logWarning(warningPrefix, "Secret store returned null for certificate"); + + List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate); + + if (x509CertificateList.isEmpty()) return logWarning(warningPrefix, "Empty certificate list"); + if (x509CertificateList.size() < 2) + return logWarning(warningPrefix, "Only a single certificate found in chain - intermediate certificates likely missing"); + + Instant now = clock.instant(); + Instant firstExpiry = Instant.MAX; + for (X509Certificate x509Certificate : x509CertificateList) { + Instant notBefore = x509Certificate.getNotBefore().toInstant(); + Instant notAfter = x509Certificate.getNotAfter().toInstant(); + if (now.isBefore(notBefore)) return logWarning(warningPrefix, "Certificate is not yet valid"); + if (now.isAfter(notAfter)) return logWarning(warningPrefix, "Certificate has expired"); + if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter; + } + + X509Certificate endEntityCertificate = x509CertificateList.get(0); + Set<String> subjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(endEntityCertificate).stream() + .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME)) + .map(SubjectAlternativeName::getValue).collect(Collectors.toSet()); + + if (Sets.intersection(subjectAlternativeNames, Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))).isEmpty()) { + return logWarning(warningPrefix, "No overlap between SANs in certificate and expected SANs"); + } + + return true; // All good then, hopefully + } catch (SecretNotFoundException s) { + return logWarning(warningPrefix, "Certificate not found in secret store"); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Exception thrown when verifying endpoint certificate", e); + return false; + } + } + + private static boolean logWarning(String warningPrefix, String message) { + log.log(LogLevel.WARNING, warningPrefix + message); + return false; + } + + private List<String> dnsNamesOf(ApplicationId applicationId, List<ZoneId> zones) { + List<String> endpointDnsNames = new ArrayList<>(); + + // We add first an endpoint name based on a hash of the applicationId, + // as the certificate provider requires the first CN to be < 64 characters long. + endpointDnsNames.add(Endpoint.createHashedCn(applicationId, zoneRegistry.system())); + + var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId()); + var rotationEndpoints = Endpoint.of(applicationId).wildcard(); + + var zoneLocalEndpoints = zones.stream().flatMap(zone -> Stream.of( + Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone), + Endpoint.of(applicationId).wildcard(zone) + )); + + Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints) + .map(Endpoint.EndpointBuilder::directRouting) + .map(endpoint -> endpoint.on(Endpoint.Port.tls())) + .map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system())) + .map(Endpoint::dnsName).forEach(endpointDnsNames::add); + + return Collections.unmodifiableList(endpointDnsNames); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 39faffcb869..c854c5b45bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -84,8 +84,8 @@ public class ApplicationOwnershipConfirmer extends Maintainer { deploymentMetrics.queriesPerSecond(), deploymentMetrics.writesPerSecond())); } - // TODO jonmv: Default instance should really be replaced with something better. - return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), metrics); + return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), + app.latestVersion().flatMap(version -> version.buildTime()), metrics); } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index bd8faaed2e2..199e9e4e7ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -49,6 +49,9 @@ public class CloudEventReporter extends Maintainer { for (var awsRegion : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { + log.info(String.format("Retrieved event %s, affecting the following instances: %s", + event.instanceEventId, + event.affectedInstances)); List<String> deprovisionedHosts = deprovisionHosts(awsRegion, event); submitIssue(event, deprovisionedHosts); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 83173fc32a7..23e3149ec1e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -27,7 +27,7 @@ import java.util.logging.Logger; */ public class JobRunner extends Maintainer { - static final Duration jobTimeout = Duration.ofDays(1); + public static final Duration jobTimeout = Duration.ofDays(1).plusHours(1); private static final Logger log = Logger.getLogger(JobRunner.class.getName()); private final JobController jobs; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java index 9c3c1dc1f5e..a814f62cb03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; @@ -75,7 +76,8 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { // another controller instance is running this job at the moment; ok } catch (Throwable t) { - log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", t); + log.log(Level.WARNING, "Maintainer " + this.getClass().getSimpleName() + " failed. Will retry in " + + maintenanceInterval + ": " + Exceptions.toMessageString(t)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index ccb802d314a..d88469645b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -64,8 +64,8 @@ public class MetricsReporter extends Maintainer { } private void reportRemainingRotations() { - try (RotationLock lock = controller().applications().rotationRepository().lock()) { - int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size(); + try (RotationLock lock = controller().routingController().rotations().lock()) { + int availableRotations = controller().routingController().rotations().availableRotations(lock).size(); metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index ac5612b4e3f..bdfaa9c098f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -82,7 +82,7 @@ public class RotationStatusUpdater extends Maintainer { private RotationStatus getStatus(Instance instance) { var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>(); for (var assignedRotation : instance.rotations()) { - var rotation = applications.rotationRepository().getRotation(assignedRotation.rotationId()); + var rotation = controller().routingController().rotations().getRotation(assignedRotation.rotationId()); if (rotation.isEmpty()) continue; var targets = service.getHealthStatus(rotation.get().name()).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java deleted file mode 100644 index ee38b2c9516..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates routing policies and their associated DNS records based on an deployment's load balancers. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicies { - - private static final Logger LOGGER = Logger.getLogger(RoutingPolicies.class.getName()); - - private final Controller controller; - private final CuratorDb db; - - public RoutingPolicies(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.db = controller.curator(); - try (var lock = db.lockRoutingPolicies()) { // Update serialized format - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - /** Read all known routing policies for given instance */ - public Set<RoutingPolicy> get(ApplicationId application) { - return db.readRoutingPolicies(application); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(DeploymentId deployment) { - return get(deployment.applicationId(), deployment.zoneId()); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) { - return db.readRoutingPolicies(application).stream() - .filter(policy -> policy.zone().equals(zone)) - .collect(Collectors.toUnmodifiableSet()); - } - - /** - * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if - * load balancers for given application have changed. - */ - public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; - var lbs = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer().getLoadBalancers(application, zone), - deploymentSpec); - try (var lock = db.lockRoutingPolicies()) { - removeObsoleteEndpointsFromDns(lbs, lock); - storePoliciesOf(lbs, lock); - removeObsoletePolicies(lbs, lock); - registerEndpointsInDns(lbs, lock); - } - } - - /** Create global endpoints for given route, if any */ - private void registerEndpointsInDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application)); - - // Create DNS record for each routing ID - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().endpointId(), - controller.system()); - Set<AliasTarget> targets = routeEntry.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } - } - - /** Store routing policies for given route. Returns the persisted policies. */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashSet<>(get(loadBalancers.application)); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var endpointIds = loadBalancers.endpointIdsOf(loadBalancer); - var policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer, endpointIds); - if (!policies.add(policy)) { - // Update existing policy - policies.remove(policy); - policies.add(policy); - } - } - db.writeRoutingPolicies(loadBalancers.application, policies); - } - - /** Create a policy for given load balancer and register a CNAME for it */ - private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer, - Set<EndpointId> endpointIds) { - var routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), - loadBalancer.dnsZone(), endpointIds, isActive(loadBalancer)); - var name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); - var data = RecordData.fqdn(loadBalancer.hostname().value()); - controller.nameServiceForwarder().createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies for given route and their CNAME records */ - private void removeObsoletePolicies(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); - var removalCandidates = new HashSet<>(allPolicies); - var activeLoadBalancers = loadBalancers.list.stream() - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Remove active load balancers and irrelevant zones from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || - !policy.zone().equals(loadBalancers.zone)); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - allPolicies.remove(policy); - } - db.writeRoutingPolicies(loadBalancers.application, allPolicies); - } - - /** Remove unreferenced global endpoints for given route from DNS */ - private void removeObsoleteEndpointsFromDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone); - var removalCandidates = routingTableFrom(zonePolicies).keySet(); - var activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - var endpoint = RoutingPolicy.endpointOf(id.application(), id.endpointId(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { - routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) { - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); - for (var policy : routingPolicies) { - for (var rotation : policy.endpoints()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.putIfAbsent(id, new ArrayList<>()); - routingTable.get(id).add(policy); - } - } - return routingTable; - } - - private static boolean isActive(LoadBalancer loadBalancer) { - switch (loadBalancer.state()) { - case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early - // as possible - case active: return true; - } - return false; - } - - /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { - - private final ApplicationId application; - private final ZoneId zone; - private final List<LoadBalancer> list; - private final DeploymentSpec deploymentSpec; - - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, - DeploymentSpec deploymentSpec) { - this.application = application; - this.zone = zone; - this.list = List.copyOf(loadBalancers); - this.deploymentSpec = deploymentSpec; - } - - /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints - return Set.of(); - } - var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); - if (instanceSpec.isEmpty()) { - return Set.of(); - } - return instanceSpec.get().endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(zone.region())) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toSet()); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index b730b63d426..ad411a895fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -19,7 +19,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -258,15 +258,13 @@ public class ApplicationSerializer { } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { - if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) { - object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong()); - toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); - applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); - applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); - } + applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number)); + applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField))); + applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); + applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); + applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -355,10 +353,8 @@ public class ApplicationSerializer { } private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) { - if (latestVersionObject.valid()) - return Optional.of(applicationVersionFromSlime(latestVersionObject)); - - return Optional.empty(); + return Optional.of(applicationVersionFromSlime(latestVersionObject)) + .filter(version -> ! version.isUnknown()); } private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) { @@ -474,22 +470,16 @@ public class ApplicationSerializer { private ApplicationVersion applicationVersionFromSlime(Inspector object) { if ( ! object.valid()) return ApplicationVersion.unknown; OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField)); - Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - if (sourceRevision.isEmpty() || applicationBuildNumber.isEmpty()) { + if (applicationBuildNumber.isEmpty()) return ApplicationVersion.unknown; - } + + Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField)); Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = Serializers.optionalString(object.field(sourceUrlField)); Optional<String> commit = Serializers.optionalString(object.field(commitField)); - if (authorEmail.isEmpty()) - return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); - - if (compileVersion.isEmpty() || buildTime.isEmpty()) - return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get()); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 22894a084b6..ad2835e301f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -11,26 +11,28 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.apache.zookeeper.data.Stat; import java.io.IOException; import java.io.UncheckedIOException; @@ -38,7 +40,9 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -80,7 +84,8 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); - private static final Path applicationCertificateRoot = root.append("applicationCertificates"); + private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); + private static final Path endpointCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); @@ -93,6 +98,7 @@ public class CuratorDb { private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); + private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); @@ -382,7 +388,6 @@ public class CuratorDb { public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) { Path path = runsPath(id, type); - cachedHistoricRuns.remove(path); curator.set(path, asJson(runSerializer.toSlime(runs))); } @@ -485,29 +490,50 @@ public class CuratorDb { // -------------- Routing policies ---------------------------------------- - public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) { + public void writeRoutingPolicies(ApplicationId application, Map<RoutingPolicyId, RoutingPolicy> policies) { curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies))); } - public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() { + public Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> readRoutingPolicies() { return curator.getChildren(routingPoliciesRoot).stream() .map(ApplicationId::fromSerializedForm) .collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies)); } - public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) { + public Map<RoutingPolicyId, RoutingPolicy> readRoutingPolicies(ApplicationId application) { return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime)) - .orElseGet(Collections::emptySet); + .orElseGet(Map::of); } - // -------------- Application web certificates ---------------------------- + public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) { + curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy))); + } - public void writeApplicationCertificate(ApplicationId applicationId, ApplicationCertificate applicationCertificate) { - curator.set(applicationCertificatePath(applicationId), applicationCertificate.secretsKeyNamePrefix().getBytes()); + public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { + return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) + .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS)); } - public Optional<ApplicationCertificate> readApplicationCertificate(ApplicationId applicationId) { - return curator.getData(applicationCertificatePath(applicationId)).map(String::new).map(ApplicationCertificate::new); + // -------------- Application endpoint certificates ---------------------------- + + public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) { + curator.set(endpointCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata))); + } + + public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) { + Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new); + return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString); + } + + public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() { + Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = new HashMap<>(); + + for (String appIdString : curator.getChildren(endpointCertificateRoot)) { + ApplicationId applicationId = ApplicationId.fromSerializedForm(appIdString); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = readEndpointCertificateMetadata(applicationId); + allEndpointCertificateMetadata.put(applicationId, endpointCertificateMetadata.orElseThrow()); + } + return allEndpointCertificateMetadata; } // -------------- Paths --------------------------------------------------- @@ -581,6 +607,8 @@ public class CuratorDb { return routingPoliciesRoot.append(application.serializedForm()); } + private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); } + private static Path nameServiceQueuePath() { return root.append("nameServiceQueue"); } @@ -625,8 +653,8 @@ public class CuratorDb { return controllerRoot.append(hostname); } - private static Path applicationCertificatePath(ApplicationId id) { - return applicationCertificateRoot.append(id.serializedForm()); + private static Path endpointCertificatePath(ApplicationId id) { + return endpointCertificateRoot.append(id.serializedForm()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java new file mode 100644 index 00000000000..653f224a02b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -0,0 +1,96 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * (de)serializes endpoint certificate metadata + * <p> + * A copy of package com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadata, + * but will soon be extended as we need to store some more information in the controller. + * + * @author andreer + */ +public class EndpointCertificateMetadataSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final static String keyNameField = "keyName"; + private final static String certNameField = "certName"; + private final static String versionField = "version"; + private final static String requestIdField = "requestId"; + private final static String requestedDnsSansField = "requestedDnsSans"; + + public static Slime toSlime(EndpointCertificateMetadata metadata) { + Slime slime = new Slime(); + Cursor object = slime.setObject(); + object.setString(keyNameField, metadata.keyName()); + object.setString(certNameField, metadata.certName()); + object.setLong(versionField, metadata.version()); + + metadata.request_id().ifPresent(id -> object.setString(requestIdField, id)); + metadata.requestedDnsSans().ifPresent(sans -> { + Cursor cursor = object.setArray(requestedDnsSansField); + sans.forEach(cursor::addString); + }); + + return slime; + } + + public static EndpointCertificateMetadata fromSlime(Inspector inspector) { + switch (inspector.type()) { + case STRING: // TODO: Remove once all are transmitted and stored as JSON + return new EndpointCertificateMetadata( + inspector.asString() + "-key", + inspector.asString() + "-cert", + 0 + ); + case OBJECT: { + Optional<String> request_id = inspector.field(requestIdField).valid() ? + Optional.of(inspector.field(requestIdField).asString()) : + Optional.empty(); + + Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? + Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : + Optional.empty(); + + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()), + request_id, + requestedDnsSans + ); + } + + default: + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + } + } + + public static EndpointCertificateMetadata fromTlsSecretsKeysString(String tlsSecretsKeys) { + return fromSlime(new Slime().setString(tlsSecretsKeys)); + } + + public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) { + if (zkdata.strip().startsWith("{")) { + return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); + } else { + return fromTlsSecretsKeysString(zkdata); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index e32e8ceacca..fffe781e6e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -6,7 +6,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type; import com.yahoo.vespa.hosted.controller.deployment.Step; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 54a3ef7551a..2429c5ee8c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,13 +6,20 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; +import java.time.Instant; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Set; +import java.util.Map; /** * Serializer and deserializer for a {@link RoutingPolicy}. @@ -35,45 +42,64 @@ public class RoutingPolicySerializer { private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; private static final String rotationsField = "rotations"; - private static final String activeField = "active"; + private static final String loadBalancerActiveField = "active"; + private static final String globalRoutingField = "globalRouting"; + private static final String agentField = "agent"; + private static final String changedAtField = "changedAt"; + private static final String statusField = "status"; - public Slime toSlime(Set<RoutingPolicy> routingPolicies) { + public Slime toSlime(Map<RoutingPolicyId, RoutingPolicy> routingPolicies) { var slime = new Slime(); var root = slime.setObject(); var policyArray = root.setArray(routingPoliciesField); - routingPolicies.forEach(policy -> { + routingPolicies.values().forEach(policy -> { var policyObject = policyArray.addObject(); - policyObject.setString(clusterField, policy.cluster().value()); - policyObject.setString(zoneField, policy.zone().value()); + policyObject.setString(clusterField, policy.id().cluster().value()); + policyObject.setString(zoneField, policy.id().zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); var rotationArray = policyObject.setArray(rotationsField); policy.endpoints().forEach(endpointId -> { rotationArray.addString(endpointId.id()); }); - policyObject.setBool(activeField, policy.active()); + policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); + globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField)); }); return slime; } - public Set<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { - var policies = new LinkedHashSet<RoutingPolicy>(); + public Map<RoutingPolicyId, RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { + var policies = new LinkedHashMap<RoutingPolicyId, RoutingPolicy>(); var root = slime.get(); var field = root.field(routingPoliciesField); field.traverse((ArrayTraverser) (i, inspect) -> { var endpointIds = new LinkedHashSet<EndpointId>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, endpointId) -> endpointIds.add(EndpointId.of(endpointId.asString()))); - var activeFieldInspector = inspect.field(activeField); - // TODO(mpolden): Remove field presence check after January 2020 - boolean active = !activeFieldInspector.valid() || activeFieldInspector.asBool(); - policies.add(new RoutingPolicy(owner, - ClusterSpec.Id.from(inspect.field(clusterField).asString()), - ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(canonicalNameField).asString()), - Serializers.optionalString(inspect.field(dnsZoneField)), - endpointIds, active)); + var id = new RoutingPolicyId(owner, + ClusterSpec.Id.from(inspect.field(clusterField).asString()), + ZoneId.from(inspect.field(zoneField).asString())); + policies.put(id, new RoutingPolicy(id, + HostName.from(inspect.field(canonicalNameField).asString()), + Serializers.optionalString(inspect.field(dnsZoneField)), + endpointIds, + new Status(inspect.field(loadBalancerActiveField).asBool(), + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); - return Collections.unmodifiableSet(policies); + return Collections.unmodifiableMap(policies); + } + + public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) { + object.setString(statusField, globalRouting.status().name()); + object.setString(agentField, globalRouting.agent().name()); + object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli()); + } + + public GlobalRouting globalRoutingFromSlime(Inspector object) { + if (!object.valid()) return GlobalRouting.DEFAULT_STATUS; + var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); + var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); + var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); + return new GlobalRouting(status, agent, changedAt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 8ffa6e65a42..1aa229984a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -30,6 +31,7 @@ import java.util.TreeMap; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; @@ -92,6 +94,8 @@ class RunSerializer { private static final String sourceField = "source"; private static final String lastTestRecordField = "lastTestRecord"; private static final String lastVespaLogTimestampField = "lastVespaLogTimestamp"; + private static final String noNodesDownSinceField = "noNodesDownSince"; + private static final String convergenceSummaryField = "convergenceSummary"; private static final String testerCertificateField = "testerCertificate"; Run runFromSlime(Slime slime) { @@ -129,12 +133,12 @@ class RunSerializer { steps, versionsFromSlime(runObject.field(versionsField)), Instant.ofEpochMilli(runObject.field(startField).asLong()), - Optional.of(runObject.field(endField)) - .filter(Inspector::valid) - .map(end -> Instant.ofEpochMilli(end.asLong())), + Serializers.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), runObject.field(lastTestRecordField).asLong(), Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), + Serializers.optionalInstant(runObject.field(noNodesDownSinceField)), + convergenceSummaryFrom(runObject.field(convergenceSummaryField)), Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) .map(certificate -> X509CertificateUtils.fromPem(certificate.asString()))); @@ -158,20 +162,44 @@ class RunSerializer { if ( ! versionObject.field(buildField).valid()) return ApplicationVersion.unknown; - SourceRevision revision = new SourceRevision(versionObject.field(repositoryField).asString(), - versionObject.field(branchField).asString(), - versionObject.field(commitField).asString()); long buildNumber = versionObject.field(buildField).asLong(); + // TODO jonmv: Remove source revision + Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(), + versionObject.field(branchField).asString(), + versionObject.field(commitField).asString())) + .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank()); Optional<String> authorEmail = Serializers.optionalString(versionObject.field(authorEmailField)); Optional<Version> compileVersion = Serializers.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = Serializers.optionalInstant(versionObject.field(buildTimeField)); Optional<String> sourceUrl = Serializers.optionalString(versionObject.field(sourceUrlField)); Optional<String> commit = Serializers.optionalString(versionObject.field(commitField)); - return new ApplicationVersion(Optional.of(revision), OptionalLong.of(buildNumber), authorEmail, + return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit); } + // Don't change this — introduce a separate array instead. + private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) { + if ( ! summaryArray.valid() || summaryArray.entries() == 11) // TODO jonmv: fix + return Optional.empty(); + + if (summaryArray.entries() != 12) + throw new IllegalArgumentException("Convergence summary must have 12 entries"); + + return Optional.of(new ConvergenceSummary(summaryArray.entry(0).asLong(), + summaryArray.entry(1).asLong(), + summaryArray.entry(2).asLong(), + summaryArray.entry(3).asLong(), + summaryArray.entry(4).asLong(), + summaryArray.entry(5).asLong(), + summaryArray.entry(6).asLong(), + summaryArray.entry(7).asLong(), + summaryArray.entry(8).asLong(), + summaryArray.entry(9).asLong(), + summaryArray.entry(10).asLong(), + summaryArray.entry(11).asLong())); + } + Slime toSlime(Iterable<Run> runs) { Slime slime = new Slime(); Cursor runArray = slime.setArray(); @@ -194,6 +222,8 @@ class RunSerializer { runObject.setString(statusField, valueOf(run.status())); runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); + run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); + run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); Cursor stepsObject = runObject.setObject(stepsField); @@ -217,12 +247,11 @@ class RunSerializer { private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); - if ( ! applicationVersion.isUnknown()) { - versionsObject.setString(repositoryField, applicationVersion.source().get().repository()); - versionsObject.setString(branchField, applicationVersion.source().get().branch()); - versionsObject.setString(commitField, applicationVersion.source().get().commit()); - versionsObject.setLong(buildField, applicationVersion.buildNumber().getAsLong()); - } + applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number)); + // TODO jonmv: Remove source revision. + applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository)); + applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch)); + applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit)); applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); @@ -230,6 +259,22 @@ class RunSerializer { applicationVersion.commit().ifPresent(commit -> versionsObject.setString(commitField, commit)); } + // Don't change this — introduce a separate array with new values if needed. + private void toSlime(ConvergenceSummary summary, Cursor summaryArray) { + summaryArray.addLong(summary.nodes()); + summaryArray.addLong(summary.down()); + summaryArray.addLong(summary.upgradingOs()); + summaryArray.addLong(summary.upgradingFirmware()); + summaryArray.addLong(summary.needPlatformUpgrade()); + summaryArray.addLong(summary.upgradingPlatform()); + summaryArray.addLong(summary.needReboot()); + summaryArray.addLong(summary.rebooting()); + summaryArray.addLong(summary.needRestart()); + summaryArray.addLong(summary.restarting()); + summaryArray.addLong(summary.services()); + summaryArray.addLong(summary.needNewConfig()); + } + static String valueOf(Step step) { switch (step) { case deployInitialReal : return "deployInitialReal"; @@ -302,14 +347,15 @@ class RunSerializer { static String valueOf(RunStatus status) { switch (status) { - case running : return "running"; - case outOfCapacity : return "outOfCapacity"; - case deploymentFailed : return "deploymentFailed"; - case installationFailed : return "installationFailed"; - case testFailure : return "testFailure"; - case error : return "error"; - case success : return "success"; - case aborted : return "aborted"; + case running : return "running"; + case outOfCapacity : return "outOfCapacity"; + case endpointCertificateTimeout : return "endpointCertificateTimeout"; + case deploymentFailed : return "deploymentFailed"; + case installationFailed : return "installationFailed"; + case testFailure : return "testFailure"; + case error : return "error"; + case success : return "success"; + case aborted : return "aborted"; default: throw new AssertionError("No value defined for '" + status + "'!"); } @@ -317,14 +363,15 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { - case "running" : return running; - case "outOfCapacity" : return outOfCapacity; - case "deploymentFailed" : return deploymentFailed; - case "installationFailed" : return installationFailed; - case "testFailure" : return testFailure; - case "error" : return error; - case "success" : return success; - case "aborted" : return aborted; + case "running" : return running; + case "outOfCapacity" : return outOfCapacity; + case "endpointCertificateTimeout" : return endpointCertificateTimeout; + case "deploymentFailed" : return deploymentFailed; + case "installationFailed" : return installationFailed; + case "testFailure" : return testFailure; + case "error" : return error; + case "success" : return success; + case "aborted" : return aborted; default: throw new IllegalArgumentException("No run status defined by '" + status + "'!"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java index 841cb387e54..e5adccc850c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.time.Instant; import java.util.Optional; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java index 789f6393683..cfd92e8a7f4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.HashSet; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 35128466e4d..7f938885cac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -10,7 +10,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java new file mode 100644 index 00000000000..6688d16ad14 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; + +import java.util.Objects; + +/** + * Serializer for {@link ZoneRoutingPolicy}. + * + * @author mpolden + */ +public class ZoneRoutingPolicySerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String GLOBAL_ROUTING_FIELD = "globalRouting"; + + private final RoutingPolicySerializer routingPolicySerializer; + + public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) { + this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null"); + } + + public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) { + var root = slime.get(); + return new ZoneRoutingPolicy(zone, routingPolicySerializer.globalRoutingFromSlime(root.field(GLOBAL_ROUTING_FIELD))); + } + + public Slime toSlime(ZoneRoutingPolicy policy) { + var slime = new Slime(); + var root = slime.setObject(); + routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD)); + return slime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 378013b5e6d..073306719f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; @@ -13,6 +13,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -27,10 +28,10 @@ import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -56,9 +57,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo; +import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; @@ -68,7 +72,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; @@ -81,6 +84,7 @@ import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -224,6 +228,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -427,15 +432,41 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); Slime slime = new Slime(); - Cursor array = slime.setArray(); + Cursor applicationArray = slime.setArray(); for (Application application : controller.applications().asList(tenant)) { - if (applicationName.map(application.id().application().value()::equals).orElse(true)) - for (InstanceName instance : application.instances().keySet()) - toSlime(application.id().instance(instance), array.addObject(), request); + if (applicationName.map(application.id().application().value()::equals).orElse(true)) { + Cursor applicationObject = applicationArray.addObject(); + applicationObject.setString("tenant", application.id().tenant().value()); + applicationObject.setString("application", application.id().application().value()); + applicationObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value(), + request.getUri()).toString()); + Cursor instanceArray = applicationObject.setArray("instances"); + for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() + : application.instances().keySet()) { + Cursor instanceObject = instanceArray.addObject(); + instanceObject.setString("instance", instance.value()); + instanceObject.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + instance.value(), + request.getUri()).toString()); + } + } } return new SlimeJsonResponse(slime); } + private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { + if ( ! type.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Only manually deployed zones have dev packages"); + + ZoneId zone = type.zone(controller.system()); + byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone); + return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); + } + private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value()); @@ -597,6 +628,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor nodeObject = nodesArray.addObject(); nodeObject.setString("hostname", node.hostname().value()); nodeObject.setString("state", valueOf(node.state())); + node.reservedTo().ifPresent(tenant -> nodeObject.setString("reservedTo", tenant.value())); nodeObject.setString("orchestration", valueOf(node.serviceState())); nodeObject.setString("version", node.currentVersion().toString()); nodeObject.setString("flavor", node.flavor()); @@ -641,6 +673,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { case admin: return "admin"; case content: return "content"; case container: return "container"; + case combined: return "combined"; default: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); } } @@ -728,7 +761,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); Cursor instancesArray = object.setArray("instances"); - for (Instance instance : application.instances().values()) + for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() + : application.instances().values()) toSlime(instancesArray.addObject(), status, instance, application.deploymentSpec(), request); application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString); @@ -750,6 +784,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { application.deploymentIssueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value())); } + // TODO: Eliminate duplicated code in this and toSlime(Cursor, Instance, DeploymentStatus, HttpRequest) private void toSlime(Cursor object, DeploymentStatus status, Instance instance, DeploymentSpec deploymentSpec, HttpRequest request) { object.setString("instance", instance.name().value()); @@ -792,35 +827,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } // Global endpoints - var globalEndpointUrls = new LinkedHashSet<String>(); - - // Add default global endpoints. These are backed by rotations. - instance.endpointsIn(controller.system()) - .scope(Endpoint.Scope.global) - .legacy(false) // Hide legacy names - .asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalEndpointUrls::add); - - // Per-cluster endpoints. These are backed by load balancers. - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); - for (var policy : routingPolicies) { - policy.rotationEndpointsIn(controller.system()).asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalEndpointUrls::add); - } - - var globalRotationsArray = object.setArray("globalRotations"); - globalEndpointUrls.forEach(globalRotationsArray::addString); - - // Legacy field. Identifies the first assigned rotation, if any. - instance.rotations().stream() - .map(AssignedRotation::rotationId) - .findFirst() - .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); - + globalEndpointsToSlime(object, instance); // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) @@ -850,6 +857,38 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } + private void globalEndpointsToSlime(Cursor object, Instance instance) { + var globalEndpointUrls = new LinkedHashSet<String>(); + + // Add default global endpoints. These are backed by rotations. + instance.endpointsIn(controller.system()) + .scope(Endpoint.Scope.global) + .legacy(false) // Hide legacy names + .asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalEndpointUrls::add); + + // Per-cluster endpoints. These are backed by load balancers. + var routingPolicies = controller.routingController().policies().get(instance.id()).values(); + for (var policy : routingPolicies) { + policy.globalEndpointsIn(controller.system()).asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalEndpointUrls::add); + } + + // TODO(mpolden): Remove once clients stop expecting this field + var globalRotationsArray = object.setArray("globalRotations"); + globalEndpointUrls.forEach(globalRotationsArray::addString); + + // Legacy field. Identifies the first assigned rotation, if any. + instance.rotations().stream() + .map(AssignedRotation::rotationId) + .findFirst() + .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); + } + private void toSlime(Cursor object, Instance instance, DeploymentStatus status, HttpRequest request) { Application application = status.application(); object.setString("tenant", instance.id().tenant().value()); @@ -913,30 +952,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); - // Rotation - Cursor globalRotationsArray = object.setArray("globalRotations"); - instance.endpointsIn(controller.system()) - .scope(Endpoint.Scope.global) - .legacy(false) // Hide legacy names - .asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalRotationsArray::addString); - - instance.rotations().stream() - .map(AssignedRotation::rotationId) - .findFirst() - .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); - - // Per-cluster rotations - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); - for (RoutingPolicy policy : routingPolicies) { - if (!policy.active()) continue; - policy.rotationEndpointsIn(controller.system()).asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalRotationsArray::addString); - } + // Global endpoint + globalEndpointsToSlime(object, instance); // Deployments sorted according to deployment spec List<Deployment> deployments = @@ -1041,22 +1058,64 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); - // Add endpoint(s) defined by routing policies + // Add zone endpoints defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies().get(deploymentId)) { - if (!policy.active()) continue; - Cursor endpointObject = endpointArray.addObject(); - Endpoint endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.cluster().value()); - endpointObject.setBool("tls", endpoint.tls()); - endpointObject.setString("url", endpoint.url().toString()); + for (var policy : controller.routingController().policies().get(deploymentId).values()) { + if (!policy.status().isActive()) continue; + { + var endpointObject = endpointArray.addObject(); + var endpoint = policy.endpointIn(controller.system()); + endpointObject.setString("cluster", policy.id().cluster().value()); + endpointObject.setBool("tls", endpoint.tls()); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); + } + // Add global endpoints that point to this policy + for (var endpoint : policy.globalEndpointsIn(controller.system()).asList()) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", policy.id().cluster().value()); + endpointObject.setBool("tls", endpoint.tls()); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); + } + } + // Add zone endpoints served by shared routing layer + for (var clusterAndUrl : controller.routingController().legacyZoneEndpointsOf(deploymentId).entrySet()) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", clusterAndUrl.getKey().value()); + endpointObject.setBool("tls", true); + endpointObject.setString("url", clusterAndUrl.getValue().toString()); + endpointObject.setString("scope", endpointScopeString(Endpoint.Scope.zone)); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); + } + // Add global endpoints served by shared routing layer + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + var instance = application.instances().get(deploymentId.applicationId().instance()); + if (deploymentId.zoneId().environment().isProduction()) { // Global endpoints can only point to production deployments + for (var rotation : instance.rotations()) { + var endpoints = instance.endpointsIn(controller.system(), rotation.endpointId()) + .legacy(false) + .scope(Endpoint.Scope.global) + .asList(); + for (var endpoint : endpoints) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", rotation.clusterId().value()); + endpointObject.setBool("tls", true); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); + } + } } - // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may - // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to - // ask the routing layer here + // serviceUrls contains all valid endpoints for this deployment, including global. The name of these endpoints + // may contain the cluster name (if non-default). Since the controller has no knowledge of clusters for legacy + // endpoints, we can't generate these URLs on-the-fly and we have to query the routing layer. + // TODO(mpolden): Remove this once all clients stop reading this. Cursor serviceUrlArray = response.setArray("serviceUrls"); - controller.applications().getDeploymentEndpoints(deploymentId) + controller.routingController().legacyEndpointsOf(deploymentId) .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); @@ -1067,12 +1126,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); DeploymentStatus status = controller.jobController().deploymentStatus(application); application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.applicationVersion().source(), response); - Instance instance = application.instances().get(deploymentId.applicationId().instance()); if (instance != null) { if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) toSlime(instance.rotations(), instance.rotationStatus(), deployment, response); @@ -1186,29 +1243,40 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new NotExistsException(instance + " has no deployment in " + zone); } - Inspector requestData = toSlime(request.getData()).get(); - String reason = mandatory("reason", requestData).asString(); - String agent = requireUserPrincipal(request).getName(); + // The order here matters because setGlobalRotationStatus involves an external request that may fail. + // TODO(mpolden): Set only one of these when only one kind of global endpoints are supported per zone. + var deploymentId = new DeploymentId(instance.id(), zone); + setGlobalRotationStatus(deploymentId, inService, request); + setGlobalEndpointStatus(deploymentId, inService, request); + + return new MessageResponse(String.format("Successfully set %s in %s %s service", + instance.id().toShortString(), zone, inService ? "in" : "out of")); + } + + /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */ + private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { + var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; + var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out; + controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); + } + + /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ + private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) { + var requestData = toSlime(request.getData()).get(); + var reason = mandatory("reason", requestData).asString(); + var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; long timestamp = controller.clock().instant().getEpochSecond(); - EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; - EndpointStatus endpointStatus = new EndpointStatus(status, reason, agent, timestamp); - controller.applications().setGlobalRotationStatus(new DeploymentId(instance.id(), deployment.zone()), - endpointStatus); - return new MessageResponse(String.format("Successfully set %s in %s.%s %s service", - instance.id().toShortString(), - deployment.zone().environment().value(), - deployment.zone().region().value(), - inService ? "in" : "out of")); + var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; + var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); + controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); } private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); - Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); - controller.applications().globalRotationStatus(deploymentId) + controller.routingController().globalRotationStatus(deploymentId) .forEach((endpoint, status) -> { array.addString(endpoint.upstreamName()); Cursor statusObject = array.addObject(); @@ -1217,7 +1285,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { statusObject.setString("agent", status.getAgent() == null ? "" : status.getAgent()); statusObject.setLong("timestamp", status.getEpoch()); }); - return new SlimeJsonResponse(slime); } @@ -1238,33 +1305,34 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse metering(String tenant, String application, HttpRequest request) { + Slime slime = new Slime(); Cursor root = slime.setObject(); - MeteringInfo meteringInfo = controller.serviceRegistry() + MeteringData meteringData = controller.serviceRegistry() .meteringService() - .getResourceSnapshots(TenantName.from(tenant), ApplicationName.from(application)); + .getMeteringData(TenantName.from(tenant), ApplicationName.from(application)); - ResourceAllocation currentSnapshot = meteringInfo.getCurrentSnapshot(); + ResourceAllocation currentSnapshot = meteringData.getCurrentSnapshot(); Cursor currentRate = root.setObject("currentrate"); currentRate.setDouble("cpu", currentSnapshot.getCpuCores()); currentRate.setDouble("mem", currentSnapshot.getMemoryGb()); currentRate.setDouble("disk", currentSnapshot.getDiskGb()); - ResourceAllocation thisMonth = meteringInfo.getThisMonth(); + ResourceAllocation thisMonth = meteringData.getThisMonth(); Cursor thismonth = root.setObject("thismonth"); thismonth.setDouble("cpu", thisMonth.getCpuCores()); thismonth.setDouble("mem", thisMonth.getMemoryGb()); thismonth.setDouble("disk", thisMonth.getDiskGb()); - ResourceAllocation lastMonth = meteringInfo.getLastMonth(); + ResourceAllocation lastMonth = meteringData.getLastMonth(); Cursor lastmonth = root.setObject("lastmonth"); lastmonth.setDouble("cpu", lastMonth.getCpuCores()); lastmonth.setDouble("mem", lastMonth.getMemoryGb()); lastmonth.setDouble("disk", lastMonth.getDiskGb()); - Map<ApplicationId, List<ResourceSnapshot>> history = meteringInfo.getSnapshotHistory(); + Map<ApplicationId, List<ResourceSnapshot>> history = meteringData.getSnapshotHistory(); Cursor details = root.setObject("details"); Cursor detailsCpu = details.setObject("cpu"); @@ -1335,10 +1403,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { - Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), - new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)), + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + + if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) { + String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, restPath); + return new HtmlResponse(result); + } + + Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); + ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), + deploymentId.applicationId(), + controller.zoneRegistry().getConfigServerApiUris(deploymentId.zoneId()), request.getUri()); response.setResponse(result, serviceName, restPath); return response; @@ -1640,23 +1715,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteInstance(id.instance(instanceName)); - if (controller.applications().requireApplication(id).instances().isEmpty()) + if (controller.applications().requireApplication(id).instances().isEmpty()) { + Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user + ? Optional.empty() + : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); + } return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - + DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + ZoneId.from(environment, region)); // Attempt to deactivate application even if the deployment is not known by the controller - DeploymentId deploymentId = new DeploymentId(instance.id(), ZoneId.from(environment, region)); - controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); - - return new MessageResponse("Deactivated " + deploymentId); + controller.applications().deactivate(id.applicationId(), id.zoneId()); + return new MessageResponse("Deactivated " + id); } /** Returns test config for indicated job, with production deployments of the default instance. */ @@ -1678,7 +1752,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.applications().clusterEndpoints(deployments), + controller.routingController().zoneEndpointsOf(deployments), controller.applications().contentClustersByZone(deployments))); } @@ -1734,10 +1808,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } + // TODO jonmv: This should list applications, not instances. Cursor applicationArray = object.setArray("applications"); for (Application application : applications) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - for (Instance instance : application.instances().values()) + for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() + : application.instances().values()) if (recurseOverApplications(request)) toSlime(applicationArray.addObject(), instance, status, request); else @@ -1969,6 +2045,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); } + private static boolean showOnlyProductionInstances(HttpRequest request) { + return "true".equals(request.getProperty("production")); + } + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case user: return "USER"; @@ -1994,7 +2074,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse submit(String tenant, String application, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - long projectId = Math.max(1, submitOptions.field("projectId").asLong()); + long projectId = Math.max(1, submitOptions.field("projectId").asLong()); // Absence of this means it's not a prod app :/ Optional<String> repository = optional("repository", submitOptions); Optional<String> branch = optional("branch", submitOptions); Optional<String> commit = optional("commit", submitOptions); @@ -2067,5 +2147,36 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return "UNKNOWN"; } + private static String endpointScopeString(Endpoint.Scope scope) { + switch (scope) { + case global: return "global"; + case zone: return "zone"; + } + throw new IllegalArgumentException("Unknown endpoint scope " + scope); + } + + private static String routingMethodString(RoutingMethod method) { + switch (method) { + case exclusive: return "exclusive"; + case shared: return "shared"; + } + throw new IllegalArgumentException("Unknown routing method " + method); + } + + private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { + return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) + .filter(cls::isInstance) + .map(cls::cast) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); + } + + /** Returns whether given request is by an operator */ + private static boolean isOperator(HttpRequest request) { + var securityContext = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class); + return securityContext.roles().stream() + .map(Role::definition) + .anyMatch(definition -> definition == RoleDefinition.hostedOperator); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java new file mode 100644 index 00000000000..99884875a64 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java @@ -0,0 +1,30 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +/** + * @author freva + */ +public class HtmlResponse extends HttpResponse { + + private final String content; + + public HtmlResponse(String content) { + super(200); + this.content = content; + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(content.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { return "text/html"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 8cc309c33aa..32c2d6ec3d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -5,6 +5,7 @@ import com.google.common.base.Joiner; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.DeploymentSpec.ChangeBlocker; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; @@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -40,21 +42,30 @@ import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.net.URI; +import java.time.Instant; +import java.time.format.TextStyle; import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.conservative; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.defaultPolicy; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.high; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.normal; @@ -171,6 +182,20 @@ class JobControllerApiHandlerHelper { devJobObject.setString("url", baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize().toString()); }); + Cursor jobsArray = responseObject.setArray("deployment"); + Arrays.stream(JobType.values()) + .filter(type -> type.environment().isManuallyDeployed()) + .map(devType -> new JobId(instance.id(), devType)) + .forEach(job -> { + Collection<Run> runs = controller.jobController().runs(job).descendingMap().values(); + if (runs.isEmpty()) + return; + + Cursor jobObject = jobsArray.addObject(); + jobObject.setString("jobName", job.type().jobName()); + toSlime(jobObject.setArray("runs"), runs, baseUriForJobs); + }); + return new SlimeJsonResponse(slime); } @@ -411,9 +436,11 @@ class JobControllerApiHandlerHelper { versionObject.setLong("build", version.buildNumber().getAsLong()); Cursor sourceObject = versionObject.setObject("source"); - sourceObject.setString("gitRepository", version.source().get().repository()); - sourceObject.setString("gitBranch", version.source().get().branch()); - sourceObject.setString("gitCommit", version.source().get().commit()); + version.source().ifPresent(source -> { + sourceObject.setString("gitRepository", source.repository()); + sourceObject.setString("gitBranch", source.branch()); + sourceObject.setString("gitCommit", source.commit()); + }); version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url)); version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); } @@ -429,9 +456,11 @@ class JobControllerApiHandlerHelper { .orElseThrow(() -> new IllegalStateException("Unknown run '" + runId + "'")); detailsObject.setBool("active", ! run.hasEnded()); detailsObject.setString("status", nameOf(run.status())); - jobController.updateTestLog(runId); - try { jobController.updateVespaLog(runId); } - catch (RuntimeException ignored) { } // May be perfectly fine, e.g., when logserver isn't up yet. + try { + jobController.updateTestLog(runId); + jobController.updateVespaLog(runId); + } + catch (RuntimeException ignored) { } // Return response when this fails, which it does when, e.g., logserver is booting. RunLog runLog = (after == null ? jobController.details(runId) : jobController.details(runId, Long.parseLong(after))) .orElseThrow(() -> new NotExistsException(String.format( @@ -450,11 +479,33 @@ class JobControllerApiHandlerHelper { Cursor stepCursor = stepsObject.setObject(step.name()); stepCursor.setString("status", info.status().name()); info.startTime().ifPresent(startTime -> stepCursor.setLong("startMillis", startTime.toEpochMilli())); + run.convergenceSummary().ifPresent(summary -> { + // If initial installation never succeeded, but is part of the job, summary concerns it. + // If initial succeeded, or is not part of this job, summary concerns upgrade installation. + if ( step == installInitialReal && info.status() != succeeded + || step == installReal && run.stepStatus(installInitialReal).map(status -> status == succeeded).orElse(true)) + toSlime(stepCursor.setObject("convergence"), summary); + }); }); return new SlimeJsonResponse(slime); } + private static void toSlime(Cursor summaryObject, ConvergenceSummary summary) { + summaryObject.setLong("nodes", summary.nodes()); + summaryObject.setLong("down", summary.down()); + summaryObject.setLong("needPlatformUpgrade", summary.needPlatformUpgrade()); + summaryObject.setLong("upgrading", summary.upgradingPlatform()); + summaryObject.setLong("needReboot", summary.needReboot()); + summaryObject.setLong("rebooting", summary.rebooting()); + summaryObject.setLong("needRestart", summary.needRestart()); + summaryObject.setLong("restarting", summary.restarting()); + summaryObject.setLong("upgradingOs", summary.upgradingOs()); + summaryObject.setLong("upgradingFirmware", summary.upgradingFirmware()); + summaryObject.setLong("services", summary.services()); + summaryObject.setLong("needNewConfig", summary.needNewConfig()); + } + private static void toSlime(Cursor entryArray, List<LogEntry> entries) { entries.forEach(entry -> toSlime(entryArray.addObject(), entry)); } @@ -539,12 +590,48 @@ class JobControllerApiHandlerHelper { stepObject.setBool("declared", stepStatus.isDeclared()); stepObject.setString("instance", stepStatus.instance().value()); + stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); + stepStatus.readyAt(change) + .filter(controller.clock().instant()::isBefore) + .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); + stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); + stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); + stepStatus.blockedUntil(change).ifPresent(until -> stepObject.setLong("blockedUntil", until.toEpochMilli())); + + if (stepStatus.type() == DeploymentStatus.StepType.instance) { + Cursor deployingObject = stepObject.setObject("deploying"); + if ( ! change.isEmpty()) { + change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString())); + change.application().ifPresent(version -> toSlime(deployingObject.setObject("application"), version)); + } + + Cursor latestVersionsObject = stepObject.setObject("latestVersions"); + List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker(); + latestVersionPreferablyWithNormalConfidenceAndNotNewerThanSystem(controller.versionStatus().versions()) + .ifPresent(latestPlatform -> { + Cursor latestPlatformObject = latestVersionsObject.setObject("platform"); + latestPlatformObject.setString("platform", latestPlatform.versionNumber().toFullString()); + latestPlatformObject.setLong("at", latestPlatform.committedAt().toEpochMilli()); + latestPlatformObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() + .anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); + toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); + }); + application.latestVersion().ifPresent(latestApplication -> { + Cursor latestApplicationObject = latestVersionsObject.setObject("application"); + toSlime(latestApplicationObject.setObject("application"), latestApplication); + latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli()); + latestApplicationObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream() + .anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0)); + toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions)); + }); + } + stepStatus.job().ifPresent(job -> { stepObject.setString("jobName", job.type().jobName()); - String baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + + URI baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + "/../instance/" + job.application().instance().value() + - "/job/" + job.type().jobName()).normalize().toString(); - stepObject.setString("url", baseUriForJob); + "/job/" + job.type().jobName()).normalize(); + stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); stepObject.setString("region", job.type().zone(controller.system()).value()); @@ -568,42 +655,17 @@ class JobControllerApiHandlerHelper { Cursor runObject = toRunArray.addObject(); toSlime(runObject.setObject("versions"), versions); - stepStatus.readyAt(change).ifPresent(ready -> runObject.setLong("readyAt", ready.toEpochMilli())); - stepStatus.readyAt(change) - .filter(controller.clock().instant()::isBefore) - .ifPresent(until -> runObject.setLong("delayedUntil", until.toEpochMilli())); - stepStatus.pausedUntil().ifPresent(until -> runObject.setLong("pausedUntil", until.toEpochMilli())); - stepStatus.coolingDownUntil(change).ifPresent(until -> runObject.setLong("coolingDownUntil", until.toEpochMilli())); - stepStatus.blockedUntil(change).ifPresent(until -> runObject.setLong("blockedUntil", until.toEpochMilli())); } - Cursor runsArray = stepObject.setArray("runs"); - jobStatus.runs().descendingMap().values().stream().limit(10).forEach(run -> { - Cursor runObject = runsArray.addObject(); - runObject.setLong("id", run.id().number()); - runObject.setString("url", baseUriForJob + "/run/" + run.id()); - runObject.setLong("start", run.start().toEpochMilli()); - run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); - runObject.setString("status", run.status().name()); - toSlime(runObject.setObject("versions"), run.versions()); - Cursor runStepsArray = runObject.setArray("steps"); - run.steps().forEach((step, info) -> { - Cursor runStepObject = runStepsArray.addObject(); - runStepObject.setString("name", step.name()); - runStepObject.setString("status", info.status().name()); - }); - }); + toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob); }); } - // TODO jonmv: Add latest platform and application status. - return new SlimeJsonResponse(slime); } private static void toSlime(Cursor versionObject, ApplicationVersion version) { - version.buildNumber().ifPresent(id -> versionObject.setLong("id", id)); - version.source().ifPresent(source -> versionObject.setString("commit", source.commit())); + version.buildNumber().ifPresent(id -> versionObject.setLong("build", id)); version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString())); version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url)); version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); @@ -616,5 +678,50 @@ class JobControllerApiHandlerHelper { versions.sourceApplication().ifPresent(application -> toSlime(versionsObject.setObject("sourceApplication"), application)); } -} + private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) { + blockers.forEach(blocker -> { + Cursor blockerObject = blockersArray.addObject(); + blocker.window().days().stream() + .map(day -> day.getDisplayName(TextStyle.SHORT, Locale.ENGLISH)) + .forEach(blockerObject.setArray("days")::addString); + blocker.window().hours() + .forEach(blockerObject.setArray("hours")::addLong); + blockerObject.setString("zone", blocker.window().zone().toString()); + }); + } + + private static Optional<VespaVersion> latestVersionPreferablyWithNormalConfidenceAndNotNewerThanSystem(List<VespaVersion> versions) { + int i; + for (i = versions.size(); i-- > 0; ) + if (versions.get(i).isSystemVersion()) + break; + + if (i < 0) + return Optional.empty(); + + for (int j = i; j >= 0; j--) + if (versions.get(j).confidence().equalOrHigherThan(normal)) + return Optional.of(versions.get(j)); + return Optional.of(versions.get(i)); + } + + private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) { + runs.stream().limit(10).forEach(run -> { + Cursor runObject = runsArray.addObject(); + runObject.setLong("id", run.id().number()); + runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString()); + runObject.setLong("start", run.start().toEpochMilli()); + run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); + runObject.setString("status", run.status().name()); + toSlime(runObject.setObject("versions"), run.versions()); + Cursor runStepsArray = runObject.setArray("steps"); + run.steps().forEach((step, info) -> { + Cursor runStepObject = runStepsArray.addObject(); + runStepObject.setString("name", step.name()); + runStepObject.setString("status", info.status().name()); + }); + }); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 2cadd864df1..ea3559a92ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.Path; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; @@ -70,6 +70,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { if (path.matches("/controller/v1/auditlog/")) return new AuditLogResponse(controller.auditLogger().readLog()); if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl()); if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader()); + if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().meteringService(), path.get("tenant"), path.get("month")); return notFound(path); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java new file mode 100644 index 00000000000..23e3195db01 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java @@ -0,0 +1,39 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.controller; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; + +import java.time.YearMonth; +import java.util.List; + +/** + * @author olaa + */ +public class MeteringResponse extends SlimeJsonResponse { + + public MeteringResponse(MeteringClient meteringClient, String tenantName, String month) { + super(toSlime(meteringClient, tenantName, month)); + } + + private static Slime toSlime(MeteringClient meteringClient, String tenantName, String month) { + Slime slime = new Slime(); + Cursor root = slime.setArray(); + List<ResourceSnapshot> snapshots = meteringClient.getSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month)); + snapshots.forEach(snapshot -> { + Cursor object = root.addObject(); + object.setString("applicationId", snapshot.getApplicationId().toShortString()); + object.setLong("timestamp", snapshot.getTimestamp().toEpochMilli()); + object.setString("zoneId", snapshot.getZoneId().value()); + object.setDouble("cpu", snapshot.getCpuCores()); + object.setDouble("memory", snapshot.getMemoryGb()); + object.setDouble("disk", snapshot.getDiskGb()); + }); + return slime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 600ad322f84..d27bc581f75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -115,6 +115,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor applicationObject = failingArray.addObject(); toSlime(applicationObject, id, request); applicationObject.setString("failing", firstFailing.id().type().jobName()); + applicationObject.setString("status", firstFailing.status().name()); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 1aaecb58a8d..ba974521278 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -87,6 +87,9 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { if (athenz.hasHostedOperatorAccess(identity)) roleMemberships.add(Role.hostedOperator()); + if (athenz.hasHostedSupporterAccess(identity)) + roleMemberships.add(Role.hostedSupporter()); + // Add all tenants that are accessible for this request athenz.accessibleTenants(tenants.asList(), new Credentials(principal)) .forEach(accessibleTenant -> roleMemberships.add(Role.athenzTenantAdmin(accessibleTenant.name()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index c168a057bfb..7d8c5922c3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -13,7 +13,7 @@ import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java new file mode 100644 index 00000000000..26ccecee3e6 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -0,0 +1,362 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.yolean.Exceptions; + +import java.net.URI; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and + * deployment-level. + * + * @author mpolden + */ +public class RoutingApiHandler extends AuditLoggingRequestHandler { + + private final Controller controller; + + public RoutingApiHandler(Context ctx, Controller controller) { + super(ctx, controller.auditLogger()); + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + } + + @Override + public HttpResponse auditAndHandle(HttpRequest request) { + try { + var path = new Path(request.getUri()); + switch (request.getMethod()) { + case GET: return get(path, request); + case POST: return post(path); + case DELETE: return delete(path); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse delete(Path path) { + if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, true); + if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, true); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse post(Path path) { + if (path.matches("/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return setDeploymentStatus(path, false); + if (path.matches("/routing/v1/inactive/environment/{environment}/region/{region}")) return setZoneStatus(path, false); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse get(Path path, HttpRequest request) { + if (path.matches("/routing/v1/")) return status(request.getUri()); + if (path.matches("/routing/v1/status/tenant/{tenant}")) return tenant(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path); + if (path.matches("/routing/v1/status/environment")) return environment(request); + if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse environment(HttpRequest request) { + var zones = controller.zoneRegistry().zones().all().ids(); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + var zonesArray = root.setArray("zones"); + for (var zone : zones) { + toSlime(zone, zonesArray.addObject()); + } + return new SlimeJsonResponse(slime); + } + var resources = controller.zoneRegistry().zones().all().ids().stream() + .map(zone -> zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse status(URI requestUrl) { + return new ResourceResponse(requestUrl, "status/tenant", "status/environment"); + } + + private HttpResponse tenant(Path path, HttpRequest request) { + var tenantName = tenantFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(controller.applications().asList(tenantName), null, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().asList(tenantName).stream() + .map(Application::id) + .map(TenantAndApplicationId::application) + .map(ApplicationName::value) + .map(application -> "application/" + application) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse application(Path path, HttpRequest request) { + var tenantAndApplicationId = tenantAndApplicationIdFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(tenantAndApplicationId)), null, + null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireApplication(tenantAndApplicationId).instances().keySet().stream() + .map(InstanceName::value) + .map(instance -> "instance/" + instance) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse instance(Path path, HttpRequest request) { + var instanceId = instanceFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(TenantAndApplicationId.from(instanceId))), + instanceId, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireInstance(instanceId).deployments().keySet().stream() + .map(zone -> "environment/" + zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse setZoneStatus(Path path, boolean in) { + var zone = zoneFrom(path); + if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { + var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; + controller.routingController().policies().setGlobalRoutingStatus(zone, status); + } else { + controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); + } + return new MessageResponse("Set global routing status for deployments in " + zone + " to " + + (in ? "IN" : "OUT")); + } + + private HttpResponse zone(Path path) { + var zone = zoneFrom(path); + var slime = new Slime(); + var root = slime.setObject(); + toSlime(zone, root); + return new SlimeJsonResponse(slime); + } + + private void toSlime(ZoneId zone, Cursor zoneObject) { + if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { + var zonePolicy = controller.routingController().policies().get(zone); + zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); + } else { + // Rotation status per zone only exposes in/out status, no agent or time of change. + var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); + var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out, + GlobalRouting.Agent.operator, Instant.EPOCH); + zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); + } + } + + private HttpResponse setDeploymentStatus(Path path, boolean in) { + var deployment = deploymentFrom(path); + var instance = controller.applications().requireInstance(deployment.applicationId()); + var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; + var agent = GlobalRouting.Agent.operator; // Always operator as this is an operator API + + // Set rotation status, if any assigned + if (rotationCanRouteTo(deployment.zoneId(), instance)) { + var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, "", + agent.name(), + controller.clock().instant().getEpochSecond()); + controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); + } + + // Set policy status + controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); + return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); + } + + private HttpResponse deployment(Path path) { + var slime = new Slime(); + var root = slime.setObject(); + var deploymentId = deploymentFrom(path); + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + toSlime(List.of(application), deploymentId.applicationId(), deploymentId.zoneId(), root); + return new SlimeJsonResponse(slime); + } + + private void toSlime(List<Application> applications, ApplicationId instanceId, ZoneId zoneId, Cursor root) { + var deploymentsArray = root.setArray("deployments"); + for (var application : applications) { + var instances = instanceId == null + ? application.instances().values() + : List.of(application.instances().get(instanceId.instance())); + for (var instance : instances) { + var zones = zoneId == null + ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)) + .collect(Collectors.toList()) + : List.of(zoneId); + for (var zone : zones) { + var deploymentId = new DeploymentId(instance.id(), zone); + // Include status from rotation + if (rotationCanRouteTo(zone, instance)) { + var rotationStatus = controller.routingController().globalRotationStatus(deploymentId); + // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. + var endpointStatus = rotationStatus.values().stream().findFirst(); + if (endpointStatus.isPresent()) { + var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); + GlobalRouting.Agent agent; + try { + agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); + } catch (IllegalArgumentException e) { + agent = GlobalRouting.Agent.unknown; + } + var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in + ? GlobalRouting.Status.in + : GlobalRouting.Status.out; + deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, + new GlobalRouting(status, agent, changedAt), + RoutingMethod.shared); + } + } + + // Include status from routing policies + var routingPolicies = controller.routingController().policies().get(deploymentId); + for (var policy : routingPolicies.values()) { + deploymentStatusToSlime(deploymentsArray.addObject(), policy); + } + } + } + } + + } + + /** Returns whether instance has an assigned rotation and a deployment in given zone */ + private static boolean rotationCanRouteTo(ZoneId zone, Instance instance) { + return !instance.rotations().isEmpty() && instance.deployments().containsKey(zone); + } + + private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) { + object.setString("routingMethod", asString(method)); + object.setString("environment", zone.environment().value()); + object.setString("region", zone.region().value()); + object.setString("status", asString(globalRouting.status())); + object.setString("agent", asString(globalRouting.agent())); + object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); + } + + private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingMethod method) { + object.setString("routingMethod", asString(method)); + object.setString("instance", deployment.applicationId().serializedForm()); + object.setString("environment", deployment.zoneId().environment().value()); + object.setString("region", deployment.zoneId().region().value()); + object.setString("status", asString(globalRouting.status())); + object.setString("agent", asString(globalRouting.agent())); + object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); + } + + private static void deploymentStatusToSlime(Cursor object, RoutingPolicy policy) { + deploymentStatusToSlime(object, new DeploymentId(policy.id().owner(), policy.id().zone()), + policy.status().globalRouting(), RoutingMethod.exclusive); + } + + private TenantName tenantFrom(Path path) { + return TenantName.from(path.get("tenant")); + } + + private ApplicationName applicationFrom(Path path) { + return ApplicationName.from(path.get("application")); + } + + private TenantAndApplicationId tenantAndApplicationIdFrom(Path path) { + return TenantAndApplicationId.from(tenantFrom(path), applicationFrom(path)); + } + + private ApplicationId instanceFrom(Path path) { + return ApplicationId.from(tenantFrom(path), applicationFrom(path), InstanceName.from(path.get("instance"))); + } + + private DeploymentId deploymentFrom(Path path) { + return new DeploymentId(instanceFrom(path), zoneFrom(path)); + } + + private ZoneId zoneFrom(Path path) { + var zone = ZoneId.from(path.get("environment"), path.get("region")); + if (!controller.zoneRegistry().hasZone(zone)) { + throw new IllegalArgumentException("No such zone: " + zone); + } + return zone; + } + + private static boolean isRecursive(HttpRequest request) { + return "true".equals(request.getProperty("recursive")); + } + + private static String asString(GlobalRouting.Status status) { + switch (status) { + case in: return "in"; + case out: return "out"; + default: return "unknown"; + } + } + + private static String asString(GlobalRouting.Agent agent) { + switch (agent) { + case operator: return "operator"; + case system: return "system"; + case tenant: return "tenant"; + default: return "unknown"; + } + } + + private static String asString(RoutingMethod method) { + switch (method) { + case shared: return "shared"; + case exclusive: return "exclusive"; + default: return "unknonwn"; + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java index 5cd6b32d572..2993b780dfe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.systemflags; +import ai.vespa.util.http.retry.DelayedConnectionLevelRetryHandler; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; @@ -23,7 +24,6 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; @@ -84,9 +84,11 @@ class FlagsClient { private static CloseableHttpClient createClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) { + DelayedConnectionLevelRetryHandler retryHandler = DelayedConnectionLevelRetryHandler.Builder + .withExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(20), 5) + .build(); return HttpClientBuilder.create() .setUserAgent("controller-flags-v1-client") - .setRetryHandler(new DefaultHttpRequestRetryHandler(5, /*retry on non-idempotent requests*/true)) .setSSLContext(identityProvider.getIdentitySslContext()) .setSSLHostnameVerifier(new FlagTargetsHostnameVerifier(targets)) .setDefaultRequestConfig(RequestConfig.custom() @@ -96,6 +98,7 @@ class FlagsClient { .build()) .setMaxConnPerRoute(2) .setMaxConnTotal(100) + .setRetryHandler(retryHandler) .build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index ca989d3323e..847a6c96a53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -17,7 +17,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeStream; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; @@ -134,10 +134,11 @@ public class UserApiHandler extends LoggingRequestHandler { // List of operator roles, currently only one available, but possible to extend List<Role> operatorRoles = roles.stream() - .filter(role -> role.definition().equals(RoleDefinition.hostedOperator)) + .filter(role -> role.definition().equals(RoleDefinition.hostedOperator) || + role.definition().equals(RoleDefinition.hostedSupporter)) + .sorted(Comparator.comparing(Role::definition)) .collect(Collectors.toList()); - Slime slime = new Slime(); Cursor root = slime.setObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java new file mode 100644 index 00000000000..7ea651c7bd5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java @@ -0,0 +1,89 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.time.Instant; +import java.util.Objects; + +/** + * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the + * time global routing status was last changed and who changed it. + * + * This is immutable. + * + * @author mpolden + */ +public class GlobalRouting { + + public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH); + + private final Status status; + private final Agent agent; + private final Instant changedAt; + + /** DO NOT USE. Public for serialization purposes */ + public GlobalRouting(Status status, Agent agent, Instant changedAt) { + this.status = Objects.requireNonNull(status, "status must be non-null"); + this.agent = Objects.requireNonNull(agent, "agent must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + } + + /** + * The wanted status of this. The system will try to set this status, but there are constraints that may lead to + * the effective status not matching this. See {@link RoutingPolicies}. + */ + public Status status() { + return status; + } + + /** The agent who last changed this */ + public Agent agent() { + return agent; + } + + /** The time this was last changed */ + public Instant changedAt() { + return changedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GlobalRouting that = (GlobalRouting) o; + return status == that.status && + agent == that.agent && + changedAt.equals(that.changedAt); + } + + @Override + public int hashCode() { + return Objects.hash(status, agent, changedAt); + } + + @Override + public String toString() { + return "status " + status + ", changed by " + agent + " @ " + changedAt; + } + + public static GlobalRouting status(Status status, Agent agent, Instant instant) { + return new GlobalRouting(status, agent, instant); + } + + // Used in serialization. Do not change. + public enum Status { + /** Status is determined by health checks **/ + in, + + /** Status is explicitly set to out */ + out, + } + + /** Agents that can change the state of global routing */ + public enum Agent { + operator, + tenant, + system, + unknown, // For compatibility old values from /routing/v1 on config server, which may contain a specific user name. + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 7b0ec3d27ba..5543d0ea0b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -1,7 +1,8 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import java.util.Objects; @@ -42,4 +43,9 @@ public class RoutingId { return Objects.hash(application, endpointId); } + @Override + public String toString() { + return "routing id for " + endpointId + " of " + application; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java new file mode 100644 index 00000000000..8657a601837 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -0,0 +1,304 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (var lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + /** Read all known routing policies for given instance */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).entrySet() + .stream() + .filter(kv -> kv.getKey().zone().equals(zone)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** Read routing policy for given zone */ + public ZoneRoutingPolicy get(ZoneId zone) { + return db.readZoneRoutingPolicy(zone); + } + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; + var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); + var inactiveZones = inactiveZones(application, deploymentSpec); + try (var lock = db.lockRoutingPolicies()) { + if (!application.instance().isTester()) removeGlobalDnsUnreferencedBy(loadBalancers, lock); + storePoliciesOf(loadBalancers, lock); + removePoliciesUnreferencedBy(loadBalancers, lock); + if (!application.instance().isTester()) updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + } + } + + /** Set the status of all global endpoints in given zone */ + public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { + try (var lock = db.lockRoutingPolicies()) { + db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, + controller.clock().instant()))); + var allPolicies = db.readRoutingPolicies(); + for (var applicationPolicies : allPolicies.values()) { + updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); + } + } + } + + /** Set the status of all global endpoints for given deployment */ + public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + try (var lock = db.lockRoutingPolicies()) { + var policies = get(deployment.applicationId()); + var newPolicies = new LinkedHashMap<>(policies); + for (var policy : policies.values()) { + if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone + var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent, + controller.clock().instant()))); + newPolicies.put(policy.id(), newPolicy); + } + db.writeRoutingPolicies(deployment.applicationId(), newPolicies); + updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + } + } + + /** Update global DNS record for given policies */ + private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + // Create DNS record for each routing ID + var routingTable = routingTableFrom(routingPolicies); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + var targets = new LinkedHashSet<AliasTarget>(); + var staleTargets = new LinkedHashSet<AliasTarget>(); + for (var policy : routeEntry.getValue()) { + if (policy.dnsZone().isEmpty()) continue; + var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); + var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + // Remove target zone if global routing status is set out at: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { + staleTargets.add(target); + } else { + targets.add(target); + } + } + // If all targets are configured out, all targets are set in. We do this because otherwise removing 100% of + // the ALIAS records would cause the global endpoint to stop resolving entirely (NXDOMAIN). + if (targets.isEmpty() && !staleTargets.isEmpty()) { + targets.addAll(staleTargets); + staleTargets.clear(); + } + if (!targets.isEmpty()) { + var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(), + routeEntry.getKey().endpointId(), controller.system()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, + RecordData.fqdn(t.name().value()), + Priority.normal)); + } + } + + /** Store routing policies for given load balancers */ + private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone); + var existingPolicy = policies.get(policyId); + var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancers.endpointIdsOf(loadBalancer), + new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); + // Preserve global routing status for existing policy + if (existingPolicy != null) { + newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting())); + } + updateZoneDnsOf(newPolicy); + policies.put(newPolicy.id(), newPolicy); + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + + /** Update zone DNS record for given policy */ + private void updateZoneDnsOf(RoutingPolicy policy) { + var name = RecordName.from(policy.endpointIn(controller.system()).dnsName()); + var data = RecordData.fqdn(policy.canonicalName().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + } + + /** Remove policies and zone DNS records unreferenced by given load balancers */ + private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = get(loadBalancers.application); + var newPolicies = new LinkedHashMap<>(policies); + var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + for (var policy : policies.values()) { + // Leave active load balancers and irrelevant zones alone + if (activeLoadBalancers.contains(policy.canonicalName()) || + !policy.id().zone().equals(loadBalancers.zone)) continue; + + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + newPolicies.remove(policy.id()); + } + db.writeRoutingPolicies(loadBalancers.application, newPolicies); + } + + /** Remove unreferenced global endpoints from DNS */ + private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values(); + var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); + var activeRoutingIds = routingIdsFrom(loadBalancers); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + + /** Compute routing IDs from given load balancers */ + private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + Set<RoutingId> routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers.list) { + for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) { + var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + for (var policy : routingPolicies) { + for (var endpoint : policy.endpoints()) { + var id = new RoutingId(policy.id().owner(), endpoint); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return Collections.unmodifiableMap(routingTable); + } + + /** Returns whether the global routing status of given policy is configured to be {@link GlobalRouting.Status#out} */ + private static boolean isConfiguredOut(RoutingPolicy policy, ZoneRoutingPolicy zonePolicy, Set<ZoneId> inactiveZones) { + // A deployment is can be configured out at any of the following levels: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + return zonePolicy.globalRouting().status() == GlobalRouting.Status.out || + policy.status().globalRouting().status() == GlobalRouting.Status.out || + inactiveZones.contains(policy.id().zone()); + } + + private static boolean isActive(LoadBalancer loadBalancer) { + switch (loadBalancer.state()) { + case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early + // as possible + case active: return true; + } + return false; + } + + /** Load balancers allocated to a deployment */ + private static class AllocatedLoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List<LoadBalancer> list; + private final DeploymentSpec deploymentSpec; + + private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + DeploymentSpec deploymentSpec) { + this.application = application; + this.zone = zone; + this.list = List.copyOf(loadBalancers); + this.deploymentSpec = deploymentSpec; + } + + /** Compute all endpoint IDs for given load balancer */ + private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { + if (!zone.environment().isProduction()) { // Only production deployments have configurable endpoints + return Set.of(); + } + var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); + if (instanceSpec.isEmpty()) { + return Set.of(); + } + return instanceSpec.get().endpoints().stream() + .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) + .filter(endpoint -> endpoint.regions().contains(zone.region())) + .map(com.yahoo.config.application.api.Endpoint::endpointId) + .map(EndpointId::of) + .collect(Collectors.toSet()); + } + + } + + /** Returns zones where global routing is declared inactive for instance through deploymentSpec */ + private static Set<ZoneId> inactiveZones(ApplicationId instance, DeploymentSpec deploymentSpec) { + var instanceSpec = deploymentSpec.instance(instance.instance()); + if (instanceSpec.isEmpty()) return Set.of(); + return instanceSpec.get().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .filter(zone -> !zone.active()) + .map(zone -> ZoneId.from(zone.environment(), zone.region().get())) + .collect(Collectors.toUnmodifiableSet()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java new file mode 100644 index 00000000000..9ef2d519c05 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -0,0 +1,106 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicy { + + private final RoutingPolicyId id; + private final HostName canonicalName; + private final Optional<String> dnsZone; + private final Set<EndpointId> endpoints; + private final Status status; + + /** DO NOT USE. Public for serialization purposes */ + public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, Set<EndpointId> endpoints, + Status status) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); + this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); + this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); + this.status = Objects.requireNonNull(status, "status must be non-null"); + } + + /** The ID of this */ + public RoutingPolicyId id() { + return id; + } + + /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ + public HostName canonicalName() { + return canonicalName; + } + + /** DNS zone for the load balancer this applies to, if any. Used when creating ALIAS records. */ + public Optional<String> dnsZone() { + return dnsZone; + } + + /** The endpoints of this policy */ + public Set<EndpointId> endpoints() { + return endpoints; + } + + /** Returns the status of this */ + public Status status() { + return status; + } + + /** Returns a copy of this with status set to given status */ + public RoutingPolicy with(Status status) { + return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status); + } + + /** Returns the endpoint of this */ + public Endpoint endpointIn(SystemName system) { + return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system); + } + + /** Returns global endpoints which this is a member of */ + public EndpointList globalEndpointsIn(SystemName system) { + return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicy that = (RoutingPolicy) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return String.format("%s [endpoints: %s%s], %s owned by %s, in %s", canonicalName, endpoints, + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(), + id.zone().value()); + } + + /** Creates a global endpoint for given application */ + public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { + return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java new file mode 100644 index 00000000000..06002e874f1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java @@ -0,0 +1,57 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Unique identifier for a {@link RoutingPolicy}. + * + * @author mpolden + */ +public class RoutingPolicyId { + + private final ApplicationId owner; + private final ClusterSpec.Id cluster; + private final ZoneId zone; + + public RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) { + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + } + + /** The application owning this */ + public ApplicationId owner() { + return owner; + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicyId that = (RoutingPolicyId) o; + return owner.equals(that.owner) && + cluster.equals(that.cluster) && + zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(owner, cluster, zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java new file mode 100644 index 00000000000..51e59c7cf4f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.util.Objects; + +/** + * Represents the status of a routing policy. + * + * This is immutable. + * + * @author mpolden + */ +public class Status { + + private final boolean active; + private final GlobalRouting globalRouting; + + /** DO NOT USE. Public for serialization purposes */ + public Status(boolean active, GlobalRouting globalRouting) { + this.active = active; + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** Returns whether this is considered active according to the load balancer status */ + public boolean isActive() { + return active; + } + + /** Return status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + /** Returns a copy of this with global routing changed */ + public Status with(GlobalRouting globalRouting) { + return new Status(active, globalRouting); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return active == status.active && + globalRouting.equals(status.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(active, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java new file mode 100644 index 00000000000..262cacd325e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}. + * + * This is immutable. + * + * @author mpolden + */ +public class ZoneRoutingPolicy { + + private final ZoneId zone; + private final GlobalRouting globalRouting; + + public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) { + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneRoutingPolicy that = (ZoneRoutingPolicy) o; + return zone.equals(that.zone) && + globalRouting.equals(that.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(zone, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index f722eb4f6bb..7c3c30738d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -1,17 +1,13 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import java.time.Instant; import java.time.ZoneOffset; -import java.util.stream.Collectors; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 1b92a3df03a..b082f66b70a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; @@ -16,7 +16,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -177,7 +178,7 @@ public class ControllerTest { public void testGlobalRotations() { // Setup ControllerTester tester = this.tester.controllerTester(); - ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()); + ZoneId zone = ZoneId.from("prod", "us-west-1"); ApplicationId app = ApplicationId.from("tenant", "app1", "default"); DeploymentId deployment = new DeploymentId(app, zone); tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( @@ -189,7 +190,7 @@ public class ControllerTest { new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1") )); - Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().applications().globalRotationStatus(deployment); + Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().routingController().globalRotationStatus(deployment); Supplier<List<EndpointStatus>> upstreamOneEndpoints = () -> { return globalRotationStatus.get() .entrySet().stream() @@ -205,21 +206,10 @@ public class ControllerTest { // Set the global rotations out of service EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond()); - tester.controller().applications().setGlobalRotationStatus(deployment, status); + tester.controller().routingController().setGlobalRotationStatus(deployment, status); assertEquals(2, upstreamOneEndpoints.get().size()); assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test"))); - - // Deployment without a global endpoint - tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( - new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3") - )); - try { - tester.controller().applications().setGlobalRotationStatus(deployment, status); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} } @Test @@ -527,9 +517,9 @@ public class ControllerTest { context.submit(applicationPackage); tester.applications().deleteApplication(context.application().id(), tester.controllerTester().credentialsFor(context.application().id().tenant())); - try (RotationLock lock = tester.applications().rotationRepository().lock()) { + try (RotationLock lock = tester.controller().routingController().rotations().lock()) { assertTrue("Rotation is unassigned", - tester.applications().rotationRepository().availableRotations(lock) + tester.controller().routingController().rotations().availableRotations(lock) .containsKey(new RotationId("rotation-id-01"))); } context.flushDnsUpdates(); @@ -592,9 +582,10 @@ public class ControllerTest { @Test public void testIntegrationTestDeployment() { Version six = Version.fromString("6.1"); - tester.controllerTester().upgradeSystem(six); tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.cd-us-central-1")); + tester.configServer().bootstrap(List.of(ZoneId.from("prod.cd-us-central-1")), SystemApplication.all()); + tester.controllerTester().upgradeSystem(six); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .majorVersion(6) @@ -709,13 +700,15 @@ public class ControllerTest { @Test public void testDeploySelectivelyProvisionsCertificate() { - Function<Instance, Optional<ApplicationCertificate>> certificate = (application) -> tester.controller().curator().readApplicationCertificate(application.id()); + Function<Instance, Optional<EndpointCertificateMetadata>> certificate = (application) -> tester.controller().curator().readEndpointCertificateMetadata(application.id()); // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod) - .region("us-west-1") - .build(); + var prodZone = ZoneId.from("prod", "us-west-1"); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); + var applicationPackage = new ApplicationPackageBuilder().environment(prodZone.environment()) + .region(prodZone.region()) + .build(); // Deploy app1 in production context1.submit(applicationPackage).deploy(); var cert = certificate.apply(context1.instance()); @@ -723,11 +716,13 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().all().ids().stream() + tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream() .flatMap(zone -> Stream.of("", "*.") - .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + ".vespa.oath.cloud"))) + .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + + ".vespa.oath.cloud"))) .collect(Collectors.toUnmodifiableList()), - tester.controllerTester().serviceRegistry().applicationCertificateMock().dnsNamesOf(context1.instanceId())); + tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId())); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); @@ -735,13 +730,13 @@ public class ControllerTest { // Create app2 var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - ZoneId zone = ZoneId.from("dev", "us-east-1"); + var devZone = ZoneId.from("dev", "us-east-1"); - // Deploy app2 in dev - tester.controller().applications().deploy(context2.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); + // Deploy app2 in a zone with shared routing + tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); assertTrue("Application deployed and activated", - tester.configServer().application(context2.instanceId(), zone).get().activated()); - assertFalse("Does not provision certificate in " + Environment.dev, certificate.apply(context2.instance()).isPresent()); + tester.configServer().application(context2.instanceId(), devZone).get().activated()); + assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent()); } @Test @@ -778,4 +773,19 @@ public class ControllerTest { } } + @Test + public void testDeployWithoutSourceRevision() { + var context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + // Submit without source revision + context.submit(applicationPackage, Optional.empty()) + .deploy(); + assertEquals("Deployed application", 1, context.instance().deployments().size()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index dbeb96337d1..a816f82c044 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; @@ -33,6 +33,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -96,6 +97,10 @@ public final class ControllerTester { new ServiceRegistryMock()); } + public ControllerTester(ServiceRegistryMock serviceRegistryMock) { + this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), serviceRegistryMock); + } + public ControllerTester(RotationsConfig rotationsConfig) { this(rotationsConfig, new MockCuratorDb()); } @@ -356,7 +361,6 @@ public final class ControllerTester { return application; } - public void deploy(ApplicationId id, ZoneId zone) { deploy(id, zone, new ApplicationPackage(new byte[0])); } @@ -394,7 +398,7 @@ public final class ControllerTester { new InMemoryFlagSource(), new MockMavenRepository(), serviceRegistry, - new MetricsMock()); + new MetricsMock(), new SecretStoreMock()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index e4b5e77b377..ed7ae12168f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -101,13 +101,19 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(RegionName regionName) { - return region(regionName.value()); + return region(regionName, true); } public ApplicationPackageBuilder region(String regionName) { - environmentBody.append(" <region active='true'>"); - environmentBody.append(regionName); - environmentBody.append("</region>\n"); + return region(RegionName.from(regionName), true); + } + + public ApplicationPackageBuilder region(RegionName regionName, boolean active) { + environmentBody.append(" <region active='") + .append(active) + .append("'>") + .append(regionName.value()) + .append("</region>\n"); return this; } @@ -195,11 +201,11 @@ public class ApplicationPackageBuilder { xml.append("'/>\n"); } xml.append(notifications); - xml.append(blockChange); if (explicitSystemTest) xml.append(" <test />\n"); if (explicitStagingTest) xml.append(" <staging />\n"); + xml.append(blockChange); xml.append(" <"); xml.append(environment.value()); if (globalServiceId != null) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java index b2a6a75b937..06d5a42f9c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BadgesTest.java @@ -26,13 +26,13 @@ public class BadgesTest { private static final ApplicationId id = ApplicationId.from("tenant", "application", "default"); private static final Run success = new Run(new RunId(id, systemTest, 3), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), - null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty()); + null, null, Optional.of(now()), RunStatus.success, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); private static final Run running = new Run(new RunId(id, systemTest, 4), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), - null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty()); + null, null, Optional.empty(), RunStatus.running, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); private static final Run failure = new Run(new RunId(id, JobType.stagingTest, 2), ImmutableMap.of(report, new StepInfo(report, Step.Status.succeeded, Optional.empty())), - null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty()); + null, null, Optional.of(now()), RunStatus.testFailure, 0, EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); @Test public void test() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 2d0b625dcb3..3e1f468691c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -1,10 +1,14 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -15,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -26,18 +31,27 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; import java.net.URI; import java.security.KeyPair; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -57,8 +71,8 @@ import static org.junit.Assert.assertTrue; * * References to this should be acquired through {@link DeploymentTester#newDeploymentContext}. * - * Tester code that is not specific to deployments should be added to either {@link ControllerTester} or - * {@link DeploymentTester} instead of this class. + * Tester code that is not specific to a single application's deployment context should be added to either + * {@link ControllerTester} or {@link DeploymentTester} instead of this class. * * @author mpolden * @author jonmv @@ -91,6 +105,7 @@ public class DeploymentContext { private final RoutingGeneratorMock routing; private final JobRunner runner; private final DeploymentTester tester; + private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private ApplicationVersion lastSubmission = null; private boolean deferDnsUpdates = false; @@ -182,6 +197,15 @@ public class DeploymentContext { return this; } + /** + * Defer provisioning of load balancers in zones in given environment. Default behaviour is to automatically + * provision load balancers in all zones that support direct routing. + */ + public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) { + deferLoadBalancerProvisioning.addAll(List.of(environment)); + return this; + } + /** Defer DNS updates */ public DeploymentContext deferDnsUpdates() { @@ -191,24 +215,54 @@ public class DeploymentContext { /** Flush all pending DNS updates */ public DeploymentContext flushDnsUpdates() { - tester.nameServiceDispatcher().run(); + flushDnsUpdates(Integer.MAX_VALUE); assertTrue("All name service requests dispatched", tester.controller().curator().readNameServiceQueue().requests().isEmpty()); return this; } + /** Flush count pending DNS updates */ + public DeploymentContext flushDnsUpdates(int count) { + var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator()), count); + dispatcher.run(); + return this; + } + + /** Add a routing policy for this in given zone, with status set to active */ + public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(instanceId, zone, active); + } + + /** Add a routing policy for tester instance of this in given zone, with status set to active */ + public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(testerId.id(), zone, active); + } + + private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) { + var clusterId = "default" + (!active ? "-inactive" : ""); + var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone); + var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance)); + policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), + Optional.empty(), + Set.of(EndpointId.of("c0")), + new Status(active, GlobalRouting.DEFAULT_STATUS))); + tester.controller().curator().writeRoutingPolicies(instance, policies); + return this; + } + /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { - return submit(applicationPackage, defaultSourceRevision); + return submit(applicationPackage, Optional.of(defaultSourceRevision)); } /** Submit given application package for deployment */ - public DeploymentContext submit(ApplicationPackage applicationPackage, SourceRevision sourceRevision) { + public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { var projectId = tester.controller().applications() .requireApplication(applicationId) .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. - lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(), + lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), Optional.empty(), projectId, applicationPackage, new byte[0]); return this; } @@ -301,7 +355,8 @@ public class DeploymentContext { if (job.type().environment().isManuallyDeployed()) return this; } - doTests(job); + if (job.type().isTest()) + doTests(job); doTeardown(job); return this; } @@ -354,11 +409,6 @@ public class DeploymentContext { return this; } - /** Sets a single endpoint in the routing layer for the instance in this */ - public DeploymentContext setEndpoints(ZoneId zone) { - return setEndpoints(zone, false); - } - /** Deploy default application package, start a run for that change and return its ID */ public RunId newRun(JobType type) { submit(); @@ -384,7 +434,6 @@ public class DeploymentContext { configServer().convergeServices(instanceId, JobType.systemTest.zone(tester.controller().system())); configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system())); setEndpoints(JobType.systemTest.zone(tester.controller().system())); - setTesterEndpoints(JobType.systemTest.zone(tester.controller().system())); runner.run(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent()); @@ -407,10 +456,20 @@ public class DeploymentContext { ZoneId zone = zone(job); DeploymentId deployment = new DeploymentId(job.application(), zone); + // Provision load balancers in directly routed zones, unless explicitly deferred + if (provisionLoadBalancerIn(zone)) { + configServer().putLoadBalancers(zone, List.of(new LoadBalancer(deployment.toString(), + deployment.applicationId(), + ClusterSpec.Id.from("default"), + HostName.from("lb-0--" + instanceId.serializedForm() + "--" + zone.toString()), + LoadBalancer.State.active, + Optional.of("dns-zone-1")))); + } + // First step is always a deployment. runner.advance(currentRun(job)); - if ( ! job.type().environment().isManuallyDeployed()) + if (job.type().isTest()) doInstallTester(job); if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. @@ -423,8 +482,7 @@ public class DeploymentContext { assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); // All installation is complete and endpoints are ready, so setup may begin. - if (job.type().isDeployment()) - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startStagingSetup)); @@ -456,17 +514,10 @@ public class DeploymentContext { return run; } - /** Sets a single endpoint in the routing layer for the tester instance in this */ - private DeploymentContext setTesterEndpoints(ZoneId zone) { - return setEndpoints(zone, true); - } - - /** Sets a single endpoint in the routing layer; this matches that required for the tester */ - private DeploymentContext setEndpoints(ZoneId zone, boolean tester) { + /** Sets a single endpoint in the routing layer */ + DeploymentContext setEndpoints(ZoneId zone) { + if (!supportsRoutingMethod(RoutingMethod.shared, zone)) return this; var id = instanceId; - if (tester) { - id = testerId.id(); - } routing.putEndpoints(new DeploymentId(id, zone), Collections.singletonList(new RoutingEndpoint(String.format("https://%s--%s--%s.%s.%s.vespa:43", id.instance().value(), @@ -512,8 +563,7 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); runner.advance(currentRun(job)); - assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); - setTesterEndpoints(zone); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); runner.advance(currentRun(job)); } @@ -547,6 +597,16 @@ public class DeploymentContext { routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone)); } + /** Returns whether a load balancer is expected to be provisioned in given zone */ + private boolean provisionLoadBalancerIn(ZoneId zone) { + return !deferLoadBalancerProvisioning.contains(zone.environment()) && + supportsRoutingMethod(RoutingMethod.exclusive, zone); + } + + private boolean supportsRoutingMethod(RoutingMethod method, ZoneId zone) { + return tester.controller().zoneRegistry().zones().routingMethod(method).ids().contains(zone); + } + private JobId jobId(JobType type) { return new JobId(instanceId, type); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5b5c6b61357..938b801b88d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -20,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest; -import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.controller.maintenance.OutstandingChangeDeployer; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; @@ -48,7 +46,6 @@ public class DeploymentTester { public static final TenantAndApplicationId appId = TenantAndApplicationId.from("tenant", "application"); public static final ApplicationId instanceId = appId.defaultInstance(); - public static final TesterId testerId = TesterId.of(instanceId); private final ControllerTester tester; private final JobController jobs; @@ -58,7 +55,6 @@ public class DeploymentTester { private final Upgrader upgrader; private final ReadyJobsTrigger readyJobsTrigger; private final OutstandingChangeDeployer outstandingChangeDeployer; - private final NameServiceDispatcher nameServiceDispatcher; public JobController jobs() { return jobs; } public RoutingGeneratorMock routing() { return routing; } @@ -92,8 +88,6 @@ public class DeploymentTester { upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); - nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), maintenanceInterval, jobControl, - Integer.MAX_VALUE); routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock. // Get deployment job logs to stderr. @@ -112,10 +106,6 @@ public class DeploymentTester { public OutstandingChangeDeployer outstandingChangeDeployer() { return outstandingChangeDeployer; } - public NameServiceDispatcher nameServiceDispatcher() { - return nameServiceDispatcher; - } - public DeploymentTester atMondayMorning() { return at(tester.clock().instant().atZone(ZoneOffset.UTC) .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 51726035cb3..db07aff34e5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -1,30 +1,31 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Before; import org.junit.Test; @@ -43,20 +44,20 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.Future; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -100,6 +101,28 @@ public class InternalStepRunnerTest { } @Test + public void retriesDeploymentForOneHour() { + RuntimeException exception = new ConfigServerException(URI.create("https://server"), + "test failure", + "Exception to retry", + ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE, + new RuntimeException("Retry me")); + tester.configServer().throwOnNextPrepare(exception); + tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + + tester.configServer().throwOnNextPrepare(exception); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + + tester.clock().advance(Duration.ofHours(1).plusSeconds(1)); + tester.configServer().throwOnNextPrepare(exception); + tester.runner().run(); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().status()); + } + + @Test public void refeedRequirementBlocksDeployment() { tester.configServer().setConfigChangeActions(new ConfigChangeActions(Collections.emptyList(), singletonList(new RefeedAction("Refeed", @@ -118,9 +141,6 @@ public class InternalStepRunnerTest { ZoneId zone = id.type().zone(system()); HostName host = tester.configServer().hostFor(instanceId, zone); - tester.setEndpoints(app.testerId().id(), JobType.productionUsCentral1.zone(system())); - tester.runner().run(); - tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster", "container", "search", @@ -143,7 +163,7 @@ public class InternalStepRunnerTest { tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(RunStatus.error, tester.jobs().run(id).get().status()); + assertEquals(installationFailed, tester.jobs().run(id).get().status()); } @Test @@ -166,61 +186,88 @@ public class InternalStepRunnerTest { tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); } @Test - public void startingTestsFailsIfDeploymentExpires() { + public void timesOutWithoutInstallationProgress() { + tester.controllerTester().upgradeSystem(new Version("7.1")); + tester.controllerTester().computeVersionStatus(); + tester.upgrader().maintain(); app.newRun(JobType.systemTest); + + // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); + tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); + tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); + tester.setEndpoints(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); + + Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), + app.instanceId()).iterator().next(); + tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(1))); + tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()), + new Node.Builder(systemTestNode) + .serviceState(Node.ServiceState.allowedDown) + .suspendedSince(tester.clock().instant()) + .build()); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); - tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); + + tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(3))); + tester.runner().run(); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + + tester.clock().advance(Duration.ofSeconds(2)); + tester.runner().run(); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); } @Test - public void startTestsFailsIfDeploymentExpires() { + public void startingTestsFailsIfDeploymentExpires() { app.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); + tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); + assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); + assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); } @Test public void alternativeEndpointsAreDetected() { + var systemTestZone = JobType.systemTest.zone(system()); + var stagingZone = JobType.stagingTest.zone(system()); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); app.newRun(JobType.systemTest); tester.runner().run();; tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - - tester.controller().curator().writeRoutingPolicies(app.instanceId(), Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); - tester.controller().curator().writeRoutingPolicies(app.testerId().id(), Set.of(new RoutingPolicy(app.testerId().id(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); + app.addRoutingPolicy(JobType.systemTest.zone(system()), true); + app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true); tester.runner().run();; assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -267,8 +314,6 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); - assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(app.testerId().id(), JobType.systemTest.zone(system()))).get(0).endpoint()), - tester.cloud().testerUrl()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString()); @@ -378,7 +423,13 @@ public class InternalStepRunnerTest { @Test public void certificateTimeoutAbortsJob() { tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd); - tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.aws-us-east-1c")); + var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); RunId id = app.startSystemTestTests(); List<X509Certificate> trusted = new ArrayList<>(publicCdApplicationPackage.trustedCertificates()); @@ -401,11 +452,45 @@ public class InternalStepRunnerTest { "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; @Test + public void generates_correct_tester_flavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <instance id='first'>\n" + + " <test tester-flavor=\"d-6-16-100\" />\n" + + " <prod>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='second'>\n" + + " <test />\n" + + " <staging />\n" + + " <prod tester-flavor=\"d-6-16-100\">\n" + + " <parallel>\n" + + " <region active=\"true\">us-east-3</region>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first")); + assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources); + + NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second")); + assertEquals(6, secondResources.vcpu(), 1e-9); + assertEquals(16, secondResources.memoryGb(), 1e-9); + assertEquals(100, secondResources.diskGb(), 1e-9); + } + + @Test public void generates_correct_services_xml_test() { - assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"), - true, - false, - new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); + assertFile("test_runner_services.xml-cd", + new String(InternalStepRunner.servicesXml( + true, + false, + new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); } private void assertFile(String resourceName, String actualContent) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index ce22e0772ff..f31038a7aba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java new file mode 100644 index 00000000000..3f8e91dec58 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java @@ -0,0 +1,108 @@ +package com.yahoo.vespa.hosted.controller.endpointcertificates; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author andreer + */ +public class EndpointCertificateManagerTest { + + private final SecretStoreMock secretStore = new SecretStoreMock(); + private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main); + private final MockCuratorDb mockCuratorDb = new MockCuratorDb(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); + private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource(); + private final Clock clock = Clock.systemUTC(); + private final EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); + + private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); + private static final X509Certificate testCertificate = X509CertificateBuilder + .fromKeypair( + testKeyPair, + new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES), + SignatureAlgorithm.SHA256_WITH_ECDSA, + X509CertificateBuilder.generateRandomSerialNumber()) + .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud") + .build(); + + private final Instance testInstance = new Instance(ApplicationId.defaultId()); + private final String testKeyName = "testKeyName"; + private final String testCertName = "testCertName"; + private ZoneId testZone; + + @Before + public void setUp() { + zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones()); + testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId(); + } + + @Test + public void provisions_new_certificate() { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, endpointCertificateMetadata.get().version()); + } + + @Test + public void reuses_stored_certificate_metadata() { + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(7, endpointCertificateMetadata.get().version()); + } + + @Test + public void uses_refreshed_certificate_when_available_and_valid() { + inMemoryFlagSource.withBooleanFlag(Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.id(), true); + + secretStore.setSecret(testKeyName, "secret-key", 7); + secretStore.setSecret(testCertName, "cert", 7); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(8, endpointCertificateMetadata.get().version()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index fef8ab32d17..5d93881129f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -18,7 +18,8 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -44,6 +45,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -62,13 +64,14 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class ConfigServerMock extends AbstractComponent implements ConfigServer { private final Map<DeploymentId, Application> applications = new LinkedHashMap<>(); + private final Set<ZoneId> inactiveZones = new HashSet<>(); private final Map<String, EndpointStatus> endpoints = new HashMap<>(); private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock(); private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>(); private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>(); private final Version initialVersion = new Version(6, 1, 0); private final Set<DeploymentId> suspendedApplications = new HashSet<>(); - private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>(); + private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); @@ -90,12 +93,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application) { + Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() + .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); nodeRepository().putByHostname(zone, new Node.Builder().hostname(hostFor(application, zone)) .state(Node.State.reserved) .type(NodeType.tenant) .owner(application) + .parentHostname(parent.hostname()) .currentVersion(initialVersion) .wantedVersion(initialVersion) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) .resources(new NodeResources(2, 8, 50, 1, slow, remote)) .serviceState(Node.ServiceState.unorchestrated) .flavor("d-2-8-50") @@ -254,8 +262,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer disallowConvergenceCheckApplications.add(applicationId); } - private List<LoadBalancer> getLoadBalancers(ZoneId zone) { - return loadBalancers.getOrDefault(zone, Collections.emptyList()); + private Set<LoadBalancer> getLoadBalancers(ZoneId zone) { + return loadBalancers.getOrDefault(zone, new LinkedHashSet<>()); } @Override @@ -275,8 +283,24 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return TesterCloud.Status.SUCCESS; } - public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { - this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); + @Override + public String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { + return "Tests started"; + } + + @Override + public List<LogEntry> getTesterLog(DeploymentId deployment, long after) { + return List.of(); + } + + @Override + public boolean isTesterReady(DeploymentId deployment) { + return false; + } + + /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ + public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { + this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>()); this.loadBalancers.get(zone).addAll(loadBalancers); } @@ -287,7 +311,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<ContainerEndpoint> containerEndpoints, - ApplicationCertificate applicationCertificate, byte[] content) { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; @@ -360,6 +384,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate"); applications.remove(deployment); serviceStatus.remove(deployment); + removeLoadBalancers(deployment.applicationId(), deployment.zoneId()); } // Returns a canned example response @@ -395,8 +420,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer // Returns a canned example response @Override - public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, - String environment, String region, String serviceName, String restPath) { + public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) { Map<String,List<?>> root = new HashMap<>(); List<Map<?,?>> resources = new ArrayList<>(); Map<String,String> resource = new HashMap<>(); @@ -407,8 +431,22 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) { - endpoints.put(endpoint, status); + public String getClusterControllerStatus(DeploymentId deployment, String restPath) { + return "<h1>OK</h1>"; + } + + @Override + public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) { + endpoints.put(upstreamName, status); + } + + @Override + public void setGlobalRotationStatus(ZoneId zone, boolean in) { + if (in) { + inactiveZones.remove(zone); + } else { + inactiveZones.add(zone); + } } @Override @@ -418,6 +456,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override + public boolean getGlobalRotationStatus(ZoneId zone) { + return !inactiveZones.contains(zone); + } + + @Override public InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters) { return new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index f99396b3b02..632b8499e11 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -132,6 +132,11 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public NodeList listNodes(ZoneId zone, List<HostName> hostnames) { + throw new UnsupportedOperationException(); + } + + @Override public List<Node> list(ZoneId zone) { return List.copyOf(nodeRepository.getOrDefault(zone, Map.of()).values()); } @@ -144,6 +149,13 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public List<Node> list(ZoneId zone, List<HostName> hostnames) { + return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream() + .filter(node -> hostnames.contains(node.hostname())) + .collect(Collectors.toList()); + } + + @Override public void upgrade(ZoneId zone, NodeType type, Version version) { this.targetVersions.compute(zone, (ignored, targetVersions) -> { if (targetVersions == null) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java index a14b7b82d67..c088965c2ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -45,4 +46,8 @@ public class SecretStoreMock extends AbstractComponent implements SecretStore { return secrets.getOrDefault(key, new TreeMap<>()).get(version); } + @Override + public List<Integer> listSecretVersions(String key) { + return List.copyOf(secrets.getOrDefault(key, new TreeMap<>()).keySet()); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index bd82807342e..323b86be1d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -42,9 +42,9 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); - private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS); + private final RoutingGeneratorMock routingGeneratorMock; private final MockMailer mockMailer = new MockMailer(); - private final ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); private final MockContactRetriever mockContactRetriever = new MockContactRetriever(); private final MockIssueHandler mockIssueHandler = new MockIssueHandler(); @@ -64,6 +64,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); this.configServerMock = new ConfigServerMock(zoneRegistryMock); + this.routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS, zoneRegistryMock); } @Inject @@ -101,8 +102,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public ApplicationCertificateMock applicationCertificateProvider() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateProvider() { + return endpointCertificateMock; } @Override @@ -212,8 +213,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return artifactRepositoryMock; } - public ApplicationCertificateMock applicationCertificateMock() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateMock() { + return endpointCertificateMock; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 269bdcc5dca..611ea4327ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import java.util.Objects; + /** * @author hakonhall */ @@ -34,6 +36,10 @@ public class ZoneApiMock implements ZoneApi { return newBuilder().with(ZoneId.from(environment, region)).build(); } + public static ZoneApiMock from(ZoneId zone) { + return newBuilder().with(zone).build(); + } + @Override public SystemName getSystemName() { return systemName; } @@ -46,6 +52,19 @@ public class ZoneApiMock implements ZoneApi { @Override public String getCloudNativeRegionName() { return cloudNativeRegionName; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneApiMock that = (ZoneApiMock) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + public static class Builder { private SystemName systemName = SystemName.defaultSystem(); private ZoneId id = ZoneId.defaultId(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 00e6162d5e5..764c2003c46 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -1,19 +1,19 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -25,20 +25,22 @@ import java.util.stream.Collectors; public class ZoneFilterMock implements ZoneList { private final List<ZoneApi> zones; + private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods; private final boolean negate; - private ZoneFilterMock(List<ZoneApi> zones, boolean negate) { + private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods, boolean negate) { this.zones = zones; + this.zoneRoutingMethods = zoneRoutingMethods; this.negate = negate; } - public static ZoneFilter from(Collection<ZoneApi> zones) { - return new ZoneFilterMock(new ArrayList<>(zones), false); + public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> routingMethods) { + return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false); } @Override public ZoneList not() { - return new ZoneFilterMock(zones, ! negate); + return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate); } @Override @@ -53,7 +55,12 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList directlyRouted() { - return all(); + return routingMethod(RoutingMethod.exclusive); + } + + @Override + public ZoneList routingMethod(RoutingMethod method) { + return filter(zone -> zoneRoutingMethods.getOrDefault(zone, Set.of()).contains(method)); } @Override @@ -63,17 +70,17 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList in(Environment... environments) { - return filter(zone -> new HashSet<>(Arrays.asList(environments)).contains(zone.getEnvironment())); + return filter(zone -> Set.of(environments).contains(zone.getEnvironment())); } @Override public ZoneList in(RegionName... regions) { - return filter(zone -> new HashSet<>(Arrays.asList(regions)).contains(zone.getRegionName())); + return filter(zone -> Set.of(regions).contains(zone.getRegionName())); } @Override public ZoneList among(ZoneId... zones) { - return filter(zone -> new HashSet<>(Arrays.asList(zones)).contains(zone.getId())); + return filter(zone -> Set.of(zones).contains(zone.getId())); } @Override @@ -93,7 +100,7 @@ public class ZoneFilterMock implements ZoneList { condition.negate().test(zone) : condition.test(zone)) .collect(Collectors.toList()), - false); + zoneRoutingMethods, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 7c44bde598c..a5e054d6068 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -1,8 +1,6 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; @@ -11,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; @@ -27,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * @author mpolden @@ -35,10 +35,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); - private List<ZoneApi> zones = List.of(); + private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); + private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + + private List<? extends ZoneApi> zones; private SystemName system; private UpgradePolicy upgradePolicy = null; - private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); /** * This sets the default list of zones contained in this. If your test need a particular set of zones, use @@ -46,15 +48,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry */ public ZoneRegistryMock(SystemName system) { this.system = system; - setZones(List.of( - ZoneApiMock.fromId("prod.aws-us-east-1a"), - ZoneApiMock.fromId("prod.ap-northeast-1"), - ZoneApiMock.fromId("prod.ap-northeast-2"), - ZoneApiMock.fromId("prod.ap-southeast-1"), - ZoneApiMock.fromId("prod.us-east-3"), - ZoneApiMock.fromId("prod.us-west-1"), - ZoneApiMock.fromId("prod.us-central-1"), - ZoneApiMock.fromId("prod.eu-west-1"))); + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), + ZoneApiMock.fromId("dev.us-east-1"), + ZoneApiMock.fromId("dev.aws-us-east-2a"), + ZoneApiMock.fromId("perf.us-east-3"), + ZoneApiMock.fromId("prod.aws-us-east-1a"), + ZoneApiMock.fromId("prod.ap-northeast-1"), + ZoneApiMock.fromId("prod.ap-northeast-2"), + ZoneApiMock.fromId("prod.ap-southeast-1"), + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1")); + // All zones use a shared routing method by default + setRoutingMethod(this.zones, RoutingMethod.shared); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -67,7 +75,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setZones(List<ZoneApi> zones) { + public ZoneRegistryMock setZones(List<? extends ZoneApi> zones) { this.zones = zones; return this; } @@ -91,6 +99,28 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } + public ZoneRegistryMock exclusiveRoutingIn(ZoneApi... zones) { + return exclusiveRoutingIn(List.of(zones)); + } + + public ZoneRegistryMock exclusiveRoutingIn(List<? extends ZoneApi> zones) { + return setRoutingMethod(zones, RoutingMethod.exclusive); + } + + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) { + return setRoutingMethod(zone, Set.of(routingMethods)); + } + + public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) { + zones.forEach(zone -> setRoutingMethod(zone, Set.of(routingMethods))); + return this; + } + + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, Set<RoutingMethod> routingMethods) { + this.zoneRoutingMethods.put(zone, Set.copyOf(routingMethods)); + return this; + } + @Override public SystemName system() { return system; @@ -98,7 +128,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public ZoneFilter zones() { - return ZoneFilterMock.from(List.copyOf(zones)); + return ZoneFilterMock.from(zones, zoneRoutingMethods); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 9de0020ce4a..c36d4494a82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -338,6 +338,25 @@ public class JobRunnerTest { } @Test + public void onlySuccessfulRunExpiresThenAnotherFails() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + var app = tester.newDeploymentContext().submit(); + JobId jobId = new JobId(app.instanceId(), systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + + app.runJob(systemTest); + assertTrue(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + + tester.clock().advance(JobController.maxHistoryAge.plusSeconds(1)); + app.submit(); + app.failDeployment(systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + } + + @Test public void timeout() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); @@ -375,12 +394,11 @@ public class JobRunnerTest { jobs.finish(jobs.last(id, systemTest).get().id()); } - Map<String, String> context = Map.of("tenant", "tenant", - "application", "real", - "instance", "default", - "job", "system-test", - "environment", "test", - "region", "us-east-1"); + Map<String, String> context = Map.of("applicationId", "tenant.real.default", + "tenantName", "tenant", + "app", "real.default", + "test", "true", + "zone", "test.us-east-1"); MetricsMock metric = ((MetricsMock) tester.controller().metric()); assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue()); @@ -389,6 +407,7 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 442a2fd1853..ceebbb8254f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -17,6 +17,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -47,7 +48,7 @@ public class OutstandingChangeDeployerTest { assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong()); - app1.submit(applicationPackage, new SourceRevision("repository1", "master", "cafed00d")); + app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d"))); assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java deleted file mode 100644 index 1bb20296bd2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Test; - -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author mortent - * @author mpolden - */ -public class RoutingPoliciesTest { - - private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); - private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); - private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); - - private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .build(); - - @Test - public void global_routing_policies() { - var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - int clustersPerZone = 2; - int numberOfDeployments = 2; - var applicationPackage = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0") - .endpoint("r1", "c0", "us-west-1") - .endpoint("r2", "c1") - .build(); - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - - // Creates alias records - context1.submit(applicationPackage).deploy(); - var endpoint1 = "r0.app1.tenant1.global.vespa.oath.cloud"; - var endpoint2 = "r1.app1.tenant1.global.vespa.oath.cloud"; - var endpoint3 = "r2.app1.tenant1.global.vespa.oath.cloud"; - - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); - assertEquals(endpoint2 + " points to c0 us-west-1", - List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint2)); - assertEquals(endpoint3 + " points to c1 in all regions", - List.of("lb-1--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-1--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint3)); - assertEquals("Routing policy count is equal to cluster count", - numberOfDeployments * clustersPerZone, - tester.policiesOf(context1.instance().id()).size()); - - // Applications gains a new deployment - ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .region(zone3.region()) - .endpoint("r0", "c0") - .endpoint("r1", "c0", "us-west-1") - .endpoint("r2", "c1") - .build(); - numberOfDeployments++; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); - context1.submit(applicationPackage2).deploy(); - - // Endpoint is updated to contain cluster in new deployment - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); - - // Another application is deployed with a single cluster and global endpoint - var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; - tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); - var applicationPackage3 = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0") - .build(); - context2.submit(applicationPackage3).deploy(); - assertEquals(endpoint4 + " points to c0 in all regions", - List.of("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint4)); - - // All endpoints for app1 are removed - ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .region(zone3.region()) - .allow(ValidationId.globalEndpointChange) - .build(); - context1.submit(applicationPackage4).deploy(); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint1)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint2)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint3)); - Set<RoutingPolicy> policies = tester.policiesOf(context1.instanceId()); - assertEquals(clustersPerZone * numberOfDeployments, policies.size()); - assertTrue("Rotation membership is removed from all policies", - policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); - assertEquals("Rotations for " + context2.application() + " are not removed", 2, tester.aliasDataOf(endpoint4).size()); - } - - @Test - public void zone_routing_policies() { - var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - - // Deploy application - int clustersPerZone = 2; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); - - // Deployment creates records and policies for all clusters in all zones - Set<String> expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context1.instanceId()).size()); - - // Next deploy does nothing - context1.submit(applicationPackage).deploy(); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context1.instanceId()).size()); - - // Add 1 cluster in each zone and deploy - tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(6, tester.policiesOf(context1.instanceId()).size()); - - // Deploy another application - tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); - context2.submit(applicationPackage).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context2.instanceId()).size()); - - // Deploy removes cluster from app1 - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - - // Remove app2 completely - tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() - .forEach(zone -> { - tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone); - tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone); - }); - context2.flushDnsUpdates(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertTrue("Removes stale routing policies " + context2.application(), tester.controllerTester().controller().applications().routingPolicies().get(context2.instanceId()).isEmpty()); - assertEquals("Keeps routing policies for " + context1.application(), 4, tester.controllerTester().controller().applications().routingPolicies().get(context1.instanceId()).size()); - } - - @Test - public void global_routing_policies_in_rotationless_system() { - var tester = new RoutingPoliciesTester(new DeploymentTester(new ControllerTester(new RotationsConfig.Builder().build()))); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - - var applicationPackage = new ApplicationPackageBuilder() - .region(zone1.region().value()) - .endpoint("r0", "c0") - .build(); - context.submit(applicationPackage).deploy(); - - var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; - assertEquals(endpoint + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint)); - assertTrue("No rotations assigned", context.application().instances().values().stream() - .map(Instance::rotations) - .allMatch(List::isEmpty)); - } - - @Test - public void cluster_endpoints_resolve_from_policies() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(3, context.instanceId(), zone1); - context.submit(applicationPackage).deploy(); - tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), Collections.emptyList()); - assertEquals(Map.of(ClusterSpec.Id.from("c0"), - URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c1"), - URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c2"), - URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), - tester.controllerTester().controller().applications().clusterEndpoints(context.deploymentIdIn(zone1))); - } - - @Test - public void manual_deployment_creates_routing_policy() { - // Empty application package is valid in manually deployed environments - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - var emptyApplicationPackage = new ApplicationPackageBuilder().build(); - var zone = ZoneId.from("dev", "us-east-1"); - tester.controllerTester().serviceRegistry().zoneRegistry().setZones(ZoneApiMock.from(zone.environment(), zone.region())); - tester.provisionLoadBalancers(1, context.instanceId(), zone); - - // Deploy to dev - tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); - assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); - context.flushDnsUpdates(); - - // Routing policy is created and DNS is updated - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals(Set.of("c0.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); - } - - @Test - public void manual_deployment_creates_routing_policy_with_non_empty_spec() { - // Initial deployment - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - context.submit(applicationPackage).deploy(); - var zone = ZoneId.from("dev", "us-east-1"); - tester.controllerTester().serviceRegistry().zoneRegistry().setZones(ZoneApiMock.from(zone.environment(), zone.region())); - - // Deploy to dev under different instance - var devInstance = context.application().id().instance("user"); - tester.provisionLoadBalancers(1, devInstance, zone); - tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); - assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); - context.flushDnsUpdates(); - - // Routing policy is created and DNS is updated - assertEquals(1, tester.policiesOf(devInstance).size()); - assertEquals(Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); - } - - @Test - public void reprovisioning_load_balancer_preserves_cname_record() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - - // Initial load balancer is provisioned - tester.provisionLoadBalancers(1, context.instanceId(), zone1); - var applicationPackage = new ApplicationPackageBuilder() - .region(zone1.region()) - .build(); - - // Application is deployed - context.submit(applicationPackage).deploy(); - var expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - - // Application is removed and the load balancer is deprovisioned - tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1); - tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); - - // Load balancer for the same application is provisioned again, but with a different hostname - var newHostname = HostName.from("new-hostname"); - var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), - context.instanceId(), - ClusterSpec.Id.from("c0"), - newHostname, - LoadBalancer.State.active, - Optional.of("dns-zone-1")); - tester.controllerTester().configServer().addLoadBalancers(zone1, List.of(loadBalancer)); - - // Application redeployment preserves DNS record - context.submit(applicationPackage).deploy(); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals("CNAME points to current load blancer", newHostname.value() + ".", - tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); - } - - private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { - List<LoadBalancer> loadBalancers = new ArrayList<>(); - for (int i = 0; i < count; i++) { - loadBalancers.add( - new LoadBalancer("LB-" + i + "-Z-" + zone.value(), - application, - ClusterSpec.Id.from("c" + i), - HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()), - LoadBalancer.State.active, - Optional.of("dns-zone-1"))); - } - return loadBalancers; - } - - private static class RoutingPoliciesTester { - - private final DeploymentTester tester; - - public RoutingPoliciesTester() { - this(new DeploymentTester()); - } - - public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { - return tester.newDeploymentContext(tenant, application, instance); - } - - public ControllerTester controllerTester() { - return tester.controllerTester(); - } - - public RoutingPoliciesTester(DeploymentTester tester) { - this.tester = tester; - } - - private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { - for (ZoneId zone : zones) { - tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); - } - } - - private Set<RoutingPolicy> policiesOf(ApplicationId instance) { - return tester.controller().curator().readRoutingPolicies(instance); - } - - private Set<String> recordNames() { - return tester.controllerTester().nameService().records().stream() - .map(Record::name) - .map(RecordName::asString) - .collect(Collectors.toSet()); - } - - private List<String> aliasDataOf(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() - .map(Record::data) - .map(RecordData::asString) - .collect(Collectors.toList()); - } - - private List<String> cnameDataOf(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() - .map(Record::data) - .map(RecordData::asString) - .collect(Collectors.toList()); - } - - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json deleted file mode 100644 index 31949cce282..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "tenant1:app1:default", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod>\n <region active=\"true\">us-central-1</region>\n <region active=\"true\">us-west-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "upgrade": false, - "at": 1506330088050 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index c4cb01f8164..dde1333eb5f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java new file mode 100644 index 00000000000..7428b9901a2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +public class EndpointCertificateMetadataSerializerTest { + + private EndpointCertificateMetadata sample = + new EndpointCertificateMetadata("keyName", "certName", 1); + private EndpointCertificateMetadata sampleWithRequestMetadata = + new EndpointCertificateMetadata("keyName", "certName", 1, "requestId", List.of("SAN1", "SAN2")); + + @Test + public void serialize() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}", + EndpointCertificateMetadataSerializer.toSlime(sample).toString()); + } + + @Test + public void serializeWithRequestMetadata() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}", + EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); + } + + @Test + public void deserializeFromString() { + assertEquals( + new EndpointCertificateMetadata("foo-key", "foo-cert", 0), + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString("foo")); + } + + @Test + public void deserializeFromJson() { + assertEquals( + sample, + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); + } + + @Test + public void deserializeFromJsonWithRequestMetadata() { + assertEquals( + sampleWithRequestMetadata, + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}")); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 23355bd6033..7ca964e06dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -1,22 +1,26 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import org.junit.Test; +import java.time.Instant; import java.util.Iterator; +import java.util.Map; import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author mortent @@ -29,41 +33,46 @@ public class RoutingPolicySerializerTest { public void serialization() { var owner = ApplicationId.defaultId(); var endpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2")); - var policies = ImmutableSet.of(new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster1"), - ZoneId.from("prod", "us-north-1"), + var id1 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster1"), + ZoneId.from("prod", "us-north-1")); + var id2 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster2"), + ZoneId.from("prod", "us-north-2")); + var policies = ImmutableMap.of(id1, new RoutingPolicy(id1, HostName.from("long-and-ugly-name"), Optional.of("zone1"), - endpoints, true), - new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster2"), - ZoneId.from("prod", "us-north-2"), + endpoints, new Status(true, GlobalRouting.DEFAULT_STATUS)), + id2, new RoutingPolicy(id2, HostName.from("long-and-ugly-name-2"), Optional.empty(), - endpoints, false)); + endpoints, new Status(false, + new GlobalRouting(GlobalRouting.Status.out, + GlobalRouting.Agent.tenant, + Instant.ofEpochSecond(123))))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); - for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext();) { + for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) { var expected = it1.next(); var actual = it2.next(); - assertEquals(expected.owner(), actual.owner()); - assertEquals(expected.cluster(), actual.cluster()); - assertEquals(expected.zone(), actual.zone()); + assertEquals(expected.id(), actual.id()); assertEquals(expected.canonicalName(), actual.canonicalName()); assertEquals(expected.dnsZone(), actual.dnsZone()); assertEquals(expected.endpoints(), actual.endpoints()); - assertEquals(expected.active(), actual.active()); + assertEquals(expected.status(), actual.status()); } } + // TODO(mpolden): Remove after January 2020 @Test public void legacy_serialization() { - var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\"," + - "\"canonicalName\":\"lb-0\"," + - "\"dnsZone\":\"dns-zone-id\",\"rotations\":[]}]}"; - var serialized = serializer.fromSlime(ApplicationId.defaultId(), SlimeUtils.jsonToSlime(json)); - assertTrue(serialized.iterator().next().active()); - + var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\",\"canonicalName\":\"lb-host\",\"dnsZone\":\"dnsZoneId\",\"rotations\":[\"default\"],\"active\":true}]}"; + var owner = ApplicationId.defaultId(); + var serialized = serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json)); + var id = new RoutingPolicyId(owner, ClusterSpec.Id.from("default"), ZoneId.from("prod", "us-north-1")); + var expected = Map.of(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.of("dnsZoneId"), + Set.of(EndpointId.defaultId()), new Status(true, GlobalRouting.DEFAULT_STATUS))); + assertEquals(expected, serialized); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index e1017173fcb..0df94598935 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -5,11 +5,12 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -75,6 +76,7 @@ public class RunSerializerTest { assertEquals(id, run.id()); assertEquals(start, run.start()); + assertEquals(Optional.of(Instant.ofEpochMilli(321321321321L)), run.noNodesDownSince()); assertFalse(run.hasEnded()); assertEquals(running, run.status()); assertEquals(3, run.lastTestLogEntry()); @@ -98,6 +100,8 @@ public class RunSerializerTest { "badb17"), 122), run.versions().sourceApplication().get()); + assertEquals(Optional.of(new ConvergenceSummary(1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144)), + run.convergenceSummary()); assertEquals(X509CertificateUtils.fromPem("-----BEGIN CERTIFICATE-----\n" + "MIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\n" + "ZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\n" + @@ -127,6 +131,7 @@ public class RunSerializerTest { run = run.with(1L << 50) .with(Instant.now().truncatedTo(MILLIS)) + .noNodesDownSince(Instant.now().truncatedTo(MILLIS)) .aborted() .finished(Instant.now().truncatedTo(MILLIS)); assertEquals(aborted, run.status()); @@ -138,6 +143,7 @@ public class RunSerializerTest { assertEquals(run.end(), phoenix.end()); assertEquals(run.status(), phoenix.status()); assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry()); + assertEquals(run.noNodesDownSince(), phoenix.noNodesDownSince()); assertEquals(run.testerCertificate(), phoenix.testerCertificate()); assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java new file mode 100644 index 00000000000..6a089c5e1b0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java @@ -0,0 +1,29 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZoneRoutingPolicySerializerTest { + + @Test + public void serialization() { + var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer()); + var zone = ZoneId.from("prod", "us-north-1"); + var policy = new ZoneRoutingPolicy(zone, + GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator, + Instant.ofEpochMilli(123))); + var serialized = serializer.fromSlime(zone, serializer.toSlime(policy)); + assertEquals(policy, serialized); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 201192280fe..a7e5d249a9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -7,6 +7,8 @@ "status": "running", "lastTestRecord": 3, "lastVespaLogTimestamp": 1196676930000432, + "noNodesDownSince": 321321321321, + "convergenceSummary": [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144], "testerCertificate": "-----BEGIN CERTIFICATE-----\nMIIBEzCBu6ADAgECAgEBMAoGCCqGSM49BAMEMBQxEjAQBgNVBAMTCW15c2Vydmlj\nZTAeFw0xOTA5MDYwNzM3MDZaFw0xOTA5MDcwNzM3MDZaMBQxEjAQBgNVBAMTCW15\nc2VydmljZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABM0JhD8fV2DlAkjQOGX3\nY50ryMBr3g2+v/uFiRoxJ1muuSOWYrW7HCQIGuzc04fa0QwtaX/voAZKCV51t6jF\n0fwwCgYIKoZIzj0EAwQDRwAwRAIgVbQ3Co1H4X0gmRrtXSyTU0HgBQu9PXHMmX20\n5MyyPSoCIBltOcmaPfdN03L3zqbqZ6PgUBWsvAHgiBzL3hrtJ+iy\n-----END CERTIFICATE-----", "steps": { "deployInitialReal": "unfinished", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index f3c437c76a1..da770c9c023 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -65,6 +65,8 @@ public class ControllerContainerTest { " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + " </rotations>\n" + " </config>\n" + + " " + + "<accesslog type='disabled'/>\n" + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + @@ -111,6 +113,9 @@ public class ControllerContainerTest { " <binding>http://*/user/v1/*</binding>\n" + " <binding>http://*/api/user/v1/*</binding>\n" + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.routing.RoutingApiHandler'>\n" + + " <binding>http://*/routing/v1/*</binding>\n" + + " </handler>\n" + variablePartXml() + "</container>"; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 30be5d9b399..88faf54ea28 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -41,7 +40,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo; +import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; @@ -52,8 +51,6 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -66,6 +63,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -77,10 +75,10 @@ import org.junit.Test; import java.io.File; import java.math.BigDecimal; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.time.YearMonth; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -97,6 +95,9 @@ import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -232,10 +233,22 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); + // GET at a tenant, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("tenant-without-applications.json")); + // GET at an application, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("application-without-instances.json")); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); @@ -250,6 +263,15 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // GET dev application package + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) + .userIdentity(USER_ID), + (response) -> { + assertEquals("attachment; filename=\"tenant1.application1.instance1.dev.us-east-1.zip\"", response.getHeaders().getFirst("Content-Disposition")); + assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); + }, + 200); + // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) @@ -483,7 +505,7 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue("Action is logged to audit log", tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin"))); + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?"))); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) @@ -559,8 +581,8 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) - deploymentTester.configServer().nodeRepository().addFixedNodes(ZoneId.from("prod", "us-central-1")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=hostA", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) + .properties(Map.of("hostname", "node-1-tenant-host-prod.us-central-1")) .screwdriverIdentity(SCREWDRIVER_ID), "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}", 200); @@ -664,7 +686,9 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // GET application package for previous build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=1", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "1")) + .userIdentity(HOSTED_VESPA_OPERATOR), (response) -> { assertEquals("attachment; filename=\"tenant1.application1-build1.zip\"", response.getHeaders().getFirst("Content-Disposition")); assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); @@ -766,16 +790,20 @@ public class ApplicationApiTest extends ControllerContainerTest { public void testRotationOverride() { // Setup createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() .instances("instance1") .globalServiceId("foo") - .region("us-west-1") - .region("us-east-3") + .region(westZone.region()) + .region(eastZone.region()) .build(); // Create tenant and deploy var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); - app.submit(applicationPackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); + app.submit(applicationPackage).deploy(); + app.addRoutingPolicy(westZone, true); + app.addRoutingPolicy(eastZone, true); // Invalid application fails tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET) @@ -784,16 +812,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // Invalid deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // Change status of non-existing deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // GET global rotation status @@ -813,11 +841,23 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-put.json")); + // Status of routing policy is changed + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + // DELETE global rotation override status tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-delete.json")); + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + + // SET global rotation override status by operator + addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", PUT) + .userIdentity(HOSTED_VESPA_OPERATOR) + .data("{\"reason\":\"unit-test\"}"), + new File("global-rotation-put.json")); + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.operator); } @Test @@ -848,19 +888,22 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // GET global rotation status for us-west-1 in default endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=default", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "default")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); // GET global rotation status for us-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"UNKNOWN\"}}", 200); // GET global rotation status for eu-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); @@ -930,7 +973,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246), ZoneId.defaultId()), new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492), ZoneId.defaultId()))); - mockMeteringClient.setMeteringInfo(new MeteringInfo(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); + mockMeteringClient.setMeteringData(new MeteringData(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET) .userIdentity(USER_ID) @@ -1075,12 +1118,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 404); // GET non-existent application package of specific build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=42", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "42")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package found for 'tenant1.application1' with build number 42\"}", 404); // GET non-existent application package of invalid build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=foobar", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "foobar")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}", 400); @@ -1352,7 +1399,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .admin(HostedAthenzIdentities.from(userId)); // POST (deploy) an application to a dev zone - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) + tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-east-1/instance/default", POST) .data(entity) .userIdentity(userId), new File("deploy-result.json")); @@ -1434,18 +1481,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-west-1") .build(); app.submit(applicationPackage).deploy(); - Set<RoutingPolicy> policies = Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-0-canonical-name"), - Optional.of("dns-zone-1"), Set.of(EndpointId.of("c0")), true), - // Inactive policy is not included - new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("deleted-cluster"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-1-canonical-name"), - Optional.of("dns-zone-1"), Set.of(), false)); - tester.controller().curator().writeRoutingPolicies(app.instanceId(), policies); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -1604,6 +1641,16 @@ public class ApplicationApiTest extends ControllerContainerTest { "queue", Optional.empty())); } + private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + var changedAt = tester.controller().clock().instant(); + var westPolicies = tester.controller().routingController().policies().get(deployment); + assertEquals(1, westPolicies.size()); + var westPolicy = westPolicies.values().iterator().next(); + assertEquals(status, westPolicy.status().globalRouting().status()); + assertEquals(agent, westPolicy.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().globalRouting().changedAt()); + } + private static class RequestBuilder implements Supplier<Request> { private final String path; @@ -1614,7 +1661,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private OktaAccessToken oktaAccessToken; private String contentType = "application/json"; private Map<String, List<String>> headers = new HashMap<>(); - private String recursive; + private Map<String, String> properties = new HashMap<>(); private RequestBuilder(String path, Request.Method method) { this.path = path; @@ -1622,7 +1669,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private RequestBuilder data(byte[] data) { this.data = data; return this; } - private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); } + private RequestBuilder data(String data) { return data(data.getBytes(UTF_8)); } private RequestBuilder data(MultiPartStreamer streamer) { return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType())); } @@ -1632,7 +1679,8 @@ public class ApplicationApiTest extends ControllerContainerTest { private RequestBuilder oktaIdentityToken(OktaIdentityToken oktaIdentityToken) { this.oktaIdentityToken = oktaIdentityToken; return this; } private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } - private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + private RequestBuilder recursive(String recursive) {return properties(Map.of("recursive", recursive)); } + private RequestBuilder properties(Map<String, String> properties) { this.properties.putAll(properties); return this; } private RequestBuilder header(String name, String value) { this.headers.putIfAbsent(name, new ArrayList<>()); this.headers.get(name).add(value); @@ -1642,11 +1690,13 @@ public class ApplicationApiTest extends ControllerContainerTest { @Override public Request get() { Request request = new Request("http://localhost:8080" + path + - // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters - (recursive == null ? "" : "?recursive=" + recursive), + properties.entrySet().stream() + .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8)) + .collect(joining("&", "?", "")), data, method); request.getHeaders().addAll(headers); request.getHeaders().put("Content-Type", contentType); + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters if (identity != null) { addIdentityToRequest(request, identity); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index a57968c11c9..1c96f46dd31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; +import java.util.Date; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; @@ -34,12 +35,10 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static org.junit.Assert.assertEquals; /** @@ -52,6 +51,9 @@ public class JobControllerApiHandlerHelperTest { public void testResponses() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .stagingTest() + .blockChange(true, true, "mon,tue", "7-13", "UTC") + .blockChange(false, true, "sun", "0-23", "CET") + .blockChange(true, false, "fri-sat", "8", "America/Los_Angeles") .region("us-central-1") .test("us-central-1") .parallel("us-west-1", "us-east-3") @@ -80,15 +82,10 @@ public class JobControllerApiHandlerHelperTest { tester.runner().run(); assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); - ZoneId usWest1 = productionUsWest1.zone(tester.controller().system()); - tester.configServer().convergeServices(app.instanceId(), usWest1); - tester.configServer().convergeServices(app.testerId().id(), usWest1); - tester.setEndpoints(app.instanceId(), usWest1); - tester.setEndpoints(app.testerId().id(), usWest1); tester.runner().run(); - tester.cloud().set(FAILURE); + tester.clock().advance(Duration.ofHours(1).plusSeconds(1)); tester.runner().run(); - assertEquals(testFailure, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); + assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); assertEquals(revision1, app.deployment(productionUsEast3.zone(tester.controller().system())).applicationVersion()); assertEquals(revision2, app.deployment(productionUsWest1.zone(tester.controller().system())).applicationVersion()); @@ -132,6 +129,7 @@ public class JobControllerApiHandlerHelperTest { // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2, // one success from revision2 to revision3 and two failures from revision1 to revision3. assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json"); @@ -139,7 +137,6 @@ public class JobControllerApiHandlerHelperTest { userApp.runJob(devAwsUsEast2a, applicationPackage); assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); - assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index 9ef94cc9afd..57774c3f412 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json new file mode 100644 index 00000000000..2479f127f92 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json @@ -0,0 +1,13 @@ +[ + { + "tenant": "tenant1", + "application":"application1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "instances": [ + { + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + } + ] + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json new file mode 100644 index 00000000000..411f9074582 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json @@ -0,0 +1,13 @@ +{ + "tenant": "tenant1", + "application": "application1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/job/", + "compileVersion": "6.1.0", + "instances": [], + "pemDeployKeys": [], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json deleted file mode 100644 index d37e9120837..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.", - "run": 1 -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index 2ab690c9d99..de6a71c14de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -6,13 +6,118 @@ "type": "instance", "dependencies": [], "declared": true, - "instance": "default" + "instance": "default", + "readyAt": 0, + "deploying": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "7.1.0", + "at": 0, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Sun" + ], + "hours": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ], + "zone": "CET" + } + ] + }, + "application": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Fri", + "Sat" + ], + "hours": [ + 8 + ], + "zone": "America/Los_Angeles" + } + ] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "default", + "readyAt": 0, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", @@ -21,24 +126,24 @@ "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/system-test/run/run 3 of system-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/system-test/run/3", + "start": 3603000, + "end": 3603000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -86,24 +191,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/system-test/run/run 2 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -151,17 +256,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/system-test/run/run 1 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -214,6 +319,9 @@ "dependencies": [], "declared": true, "instance": "default", + "readyAt": 4353000, + "delayedUntil": 4353000, + "coolingDownUntil": 4353000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -223,45 +331,42 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } - }, - "readyAt": 752000, - "delayedUntil": 752000, - "coolingDownUntil": 752000 + } } ], "runs": [ { "id": 5, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 5 of staging-test for tenant.application", - "start": 102000, - "end": 102000, + "url": "https://some.url:43/instance/default/job/staging-test/run/5", + "start": 3703000, + "end": 3703000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -325,24 +430,24 @@ }, { "id": 4, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 4 of staging-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/staging-test/run/4", + "start": 3603000, + "end": 3603000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -406,24 +511,24 @@ }, { "id": 3, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 3 of staging-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/staging-test/run/3", + "start": 3603000, + "end": 3603000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -487,24 +592,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 2 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -568,17 +673,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 1 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -650,50 +755,43 @@ ], "declared": true, "instance": "default", + "readyAt": 3603000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", "region": "prod.us-central-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [], "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 3 of production-us-central-1 for tenant.application", - "start": 2000, + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", + "start": 3603000, "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -702,55 +800,35 @@ "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "unfinished" - }, - { "name": "report", - "status": "unfinished" + "status": "succeeded" } ] }, { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 2 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -759,18 +837,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -778,29 +844,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -809,18 +867,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -844,17 +890,17 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -862,24 +908,24 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 2 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -911,24 +957,24 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 1 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -973,27 +1019,27 @@ "region": "prod.us-west-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -1001,56 +1047,36 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 2 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 1000, - "status": "testFailure", + "end": 3602000, + "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, { "name": "installReal", - "status": "succeeded" - }, - { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", "status": "failed" }, { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1058,29 +1084,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -1089,18 +1107,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1121,27 +1127,27 @@ "region": "prod.us-east-3", "currentPlatform": "6.1.0", "currentApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -1149,56 +1155,36 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 2 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/2", "start": 1000, "end": 1000, "status": "deploymentFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "failed" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", - "status": "unfinished" + "status": "failed" }, { "name": "installReal", "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1206,29 +1192,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -1237,18 +1215,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index e619e66b462..282c18046d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -6,13 +6,42 @@ "type": "instance", "dependencies": [], "declared": true, - "instance": "instance1" + "instance": "instance1", + "readyAt": 0, + "deploying": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", @@ -21,16 +50,16 @@ "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 2 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -78,17 +107,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 1 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -141,6 +170,7 @@ "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", @@ -149,16 +179,16 @@ "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 2 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -222,17 +252,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 1 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -312,10 +342,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -323,29 +353,21 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -354,18 +376,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -389,10 +399,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -400,27 +410,19 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", - "sourceUrl": "repository1/tree/commit1" + "build": 1, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "unfinished" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "unfinished" }, @@ -429,18 +431,6 @@ "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "unfinished" - }, - { "name": "report", "status": "unfinished" } @@ -464,10 +454,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -475,28 +465,20 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "unfinished" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "unfinished" }, @@ -505,18 +487,6 @@ "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "unfinished" - }, - { "name": "report", "status": "unfinished" } @@ -531,13 +501,34 @@ 5 ], "declared": true, - "instance": "instance2" + "instance": "instance2", + "deploying": {}, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/system-test", "environment": "test", @@ -550,6 +541,7 @@ "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/staging-test", "environment": "staging", @@ -573,10 +565,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -599,10 +591,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -625,10 +617,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 22ba83a4730..eb8bf523474 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -8,7 +8,23 @@ { "cluster": "default", "tls": true, - "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/" + "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "exclusive" + }, + { + "cluster": "default", + "tls": true, + "url": "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/", + "scope": "global", + "routingMethod": "exclusive" + }, + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa:43", + "scope": "zone", + "routingMethod": "shared" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index f9692a4afb7..ef8899c0860 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -4,7 +4,22 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "scope": "zone", + "routingMethod": "shared" + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global", + "routingMethod": "shared" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index fe90ddd772e..f7185d373c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 102000, - "end": 102000, + "start": 3703000, + "end": 3703000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index bfaa62a602d..b37d0d41ae4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -29,5 +29,37 @@ ], "url": "https://some.url:43/root/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 0, + "end": 0, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index e5b2fef2120..e1c2310ce7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -1,4 +1,6 @@ { + "active": true, + "status": "running", "log": { "deployReal": [ { @@ -26,27 +28,17 @@ { "at": 0, "type": "info", - "message": "Checking installation of 6.1 and unknown ..." - }, - { - "at": 0, - "type": "info", - "message": " host-tenant:application:default-dev.us-east-1: unorchestrated 6.1 " - }, - { - "at": 0, - "type": "info", - "message": "Wanted config generation is 2" + "message": "host-tenant:application:default-dev.us-east-1: unorchestrated" }, { "at": 0, "type": "info", - "message": " host-tenant:application:default-dev.us-east-1: container on port 43 has config generation 1" + "message": "--- platform 6.1" }, { "at": 0, "type": "info", - "message": "Installation not yet complete." + "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "copyVespaLogs": [ @@ -57,20 +49,32 @@ } ] }, - "active": true, - "lastId": 9, + "lastId": 7, "steps": { "deployReal": { - "startMillis": 0, - "status": "succeeded" + "status": "succeeded", + "startMillis": 0 }, "installReal": { + "status": "unfinished", "startMillis": 0, - "status": "unfinished" + "convergence": { + "nodes": 1, + "down": 0, + "needPlatformUpgrade": 0, + "upgrading": 0, + "needReboot": 0, + "rebooting": 0, + "needRestart": 0, + "restarting": 0, + "upgradingOs": 0, + "upgradingFirmware": 0, + "services": 1, + "needNewConfig": 1 + } }, "copyVespaLogs": { "status": "unfinished" } - }, - "status": "running" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index 3248bfa78a1..b63031fab4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -1,44 +1,11 @@ { + "active": false, + "status": "success", "log": { "installReal": [ { "at": 0, "type": "info", - "message": "Checking installation of 6.1 and unknown ..." - }, - { - "at": 0, - "type": "info", - "message": " host-tenant:application:default-dev.us-east-1: unorchestrated 6.1 " - }, - { - "at": 0, - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": 0, - "type": "info", - "message": "All services on wanted config generation." - }, - { - "at": 0, - "type": "info", - "message": "Attempting to find deployment endpoints ..." - }, - { - "at": 0, - "type": "info", - "message": "Found endpoints:" - }, - { - "at": 0, - "type": "info", - "message": "- dev.us-east-1" - }, - { - "at": 0, - "type": "info", "message": " |-- https://default--application--tenant.us-east-1.dev.vespa:43 (cluster 'default')" }, { @@ -48,21 +15,19 @@ } ] }, - "active": false, - "lastId": 18, + "lastId": 11, "steps": { "deployReal": { - "startMillis": 0, - "status": "succeeded" + "status": "succeeded", + "startMillis": 0 }, "installReal": { - "startMillis": 0, - "status": "succeeded" + "status": "succeeded", + "startMillis": 0 }, "copyVespaLogs": { - "startMillis": 0, - "status": "succeeded" + "status": "succeeded", + "startMillis": 0 } - }, - "status": "success" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 204927f6954..cc930c94051 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -4,7 +4,15 @@ "instance": "instance1", "environment": "dev", "region": "us-east-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa:43", + "scope": "zone", + "routingMethod": "shared" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-east-1.dev.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json deleted file mode 100644 index 024ca11dbe3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - @include(instance-reference.json) -]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json deleted file mode 100644 index 6338306897c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "projectId": 1000, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - } - ], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], - "rotationId": "rotation-id-1", - "instances": [ - { - "bcpStatus": { - "rotationStatus": "IN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "IN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-west-1", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" - }, - { - "bcpStatus": { - "rotationStatus": "UNKNOWN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "UNKNOWN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-east-3", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" - } - ], - "pemDeployKeys": [], - "metrics": { - "queryServiceQuality": 0.5, - "writeServiceQuality": 0.7 - }, - "activity": { - "lastQueried": 1527848130000, - "lastWritten": 1527848130000, - "lastQueriesPerSecond": 1.0, - "lastWritesPerSecond": 2.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index 85a3245c308..e97c76668d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -3,5 +3,6 @@ "deployments": [], "lastVersions": {}, "deploying": {}, - "jobs": {} + "jobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index dc37c3b4bb4..8cd102432d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -230,19 +230,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1" } @@ -268,13 +262,8 @@ "commit": "commit1" }, "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", "deployReal": "unfinished", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", "report": "unfinished" }, "tasks": {}, @@ -302,13 +291,8 @@ "commit": "commit1" }, "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", "deployReal": "unfinished", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", "report": "unfinished" }, "tasks": {}, @@ -344,5 +328,37 @@ ], "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index cf47e8671b0..aaa9127bdfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -1,55 +1,86 @@ { + "lastVersions": { + "platform": { + "platform": "7.1", + "at": 0, + "completed": "0 of 0 complete" + }, + "application": { + "application": { + "hash": "1.0.3-commit1", + "build": 3, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "completed": "0 of 0 complete" + } + }, + "deploying": {}, + "deployments": [], + "jobs": {}, "devJobs": { "dev-aws-us-east-2a": { "runs": [ { + "id": 1, + "status": "success", + "start": 3703000, + "end": 3703000, "wantedPlatform": "7.1", - "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1", "wantedApplication": { "hash": "unknown" }, - "start": 102000, - "end": 102000, - "id": 1, "steps": { "deployReal": "succeeded", "installReal": "succeeded", "copyVespaLogs": "succeeded" }, "tasks": { - "install": "succeeded", - "deploy": "succeeded" + "deploy": "succeeded", + "install": "succeeded" }, - "status": "success" + "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1" } ], "url": "https://some.url:43/root/dev-aws-us-east-2a" } }, - "deployments": [], - "lastVersions": { - "application": { - "at": 1000, - "application": { - "build": 3, - "source": { - "gitRepository": "repository1", - "gitCommit": "commit1", - "gitBranch": "master" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "hash": "1.0.3-commit1" - }, - "completed": "0 of 0 complete" - }, - "platform": { - "at": 0, - "completed": "0 of 0 complete", - "platform": "7.1" + "deployment": [ + { + "jobName": "dev-aws-us-east-2a", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root//run/1", + "start": 3703000, + "end": 3703000, + "status": "success", + "versions": { + "targetPlatform": "7.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] } - }, - "deploying": {}, - "jobs": {} + ] } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index ad00476154c..1e1a4549006 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -37,7 +37,7 @@ "deployments": [ { "us-central-1": { - "at": 2000, + "at": 3603000, "platform": "6.1", "application": { "hash": "1.0.3-commit1", @@ -97,8 +97,8 @@ { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 3603000, + "end": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -265,8 +265,8 @@ { "id": 5, "status": "installationFailed", - "start": 102000, - "end": 102000, + "start": 3703000, + "end": 3703000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -313,8 +313,8 @@ { "id": 4, "status": "installationFailed", - "start": 2000, - "end": 2000, + "start": 3603000, + "end": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -361,8 +361,8 @@ { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 3603000, + "end": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -510,7 +510,7 @@ { "id": 3, "status": "running", - "start": 2000, + "start": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -536,14 +536,9 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "unfinished", "deployReal": "succeeded", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", - "report": "unfinished" + "report": "succeeded" }, "tasks": { "deploy": "succeeded", @@ -581,19 +576,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-central-1/run/2" }, @@ -615,19 +604,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-central-1/run/1" } @@ -789,9 +772,9 @@ }, { "id": 2, - "status": "testFailure", + "status": "installationFailed", "start": 1000, - "end": 1000, + "end": 3602000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.2-commit1", @@ -817,19 +800,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "failed", - "deactivateTester": "succeeded", + "installReal": "failed", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "failed" + "install": "failed" }, "log": "https://some.url:43/root/production-us-west-1/run/2" }, @@ -851,19 +828,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-west-1/run/1" } @@ -933,16 +904,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "failed", - "installTester": "unfinished", - "deployReal": "unfinished", + "deployReal": "failed", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "succeeded", "report": "succeeded" }, - "tasks": {}, + "tasks": { + "deploy": "failed" + }, "log": "https://some.url:43/root/production-us-east-3/run/2" }, { @@ -963,19 +931,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-east-3/run/1" } @@ -983,5 +945,6 @@ "url": "https://some.url:43/root/production-us-east-3" } }, - "devJobs": {} + "devJobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 39824e22928..436c2767b3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -7,7 +7,22 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "scope": "zone", + "routingMethod": "shared" + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global", + "routingMethod": "shared" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 4238ce4b834..a7825681f4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 3603000, + "end": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 2000, - "end": 2000, + "start": 3603000, + "end": 3603000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 102000, - "end": 102000, + "start": 3703000, + "end": 3703000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json new file mode 100644 index 00000000000..8ade8795c86 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -0,0 +1,195 @@ +{ + "active": false, + "status": "installationFailed", + "log": { + "deployTester": [ + { + "at": 3703000, + "type": "info", + "message": "No services requiring restart." + }, + { + "at": 3703000, + "type": "info", + "message": "Deployment successful." + }, + { + "at": 3703000, + "type": "info", + "message": "foo" + } + ], + "installTester": [ + { + "at": 3703000, + "type": "info", + "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + }, + { + "at": 3703000, + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": 3703000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 3703000, + "type": "info", + "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + }, + { + "at": 3703000, + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": 3703000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 3703000, + "type": "info", + "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + }, + { + "at": 3703000, + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": 3703000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + } + ], + "deployInitialReal": [ + { + "at": 3703000, + "type": "info", + "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." + }, + { + "at": 3703000, + "type": "info", + "message": "No services requiring restart." + }, + { + "at": 3703000, + "type": "info", + "message": "Deployment successful." + }, + { + "at": 3703000, + "type": "info", + "message": "foo" + } + ], + "installInitialReal": [ + { + "at": 3703000, + "type": "info", + "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + }, + { + "at": 3703000, + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": 3703000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 3703000, + "type": "info", + "message": "Deployment expired before installation was successful." + } + ], + "deactivateReal": [ + { + "at": 3703000, + "type": "info", + "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." + } + ], + "deactivateTester": [ + { + "at": 3703000, + "type": "info", + "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." + } + ] + }, + "lastId": 22, + "steps": { + "deployTester": { + "status": "succeeded", + "startMillis": 3703000 + }, + "installTester": { + "status": "unfinished", + "startMillis": 3703000 + }, + "deployInitialReal": { + "status": "succeeded", + "startMillis": 3703000 + }, + "installInitialReal": { + "status": "failed", + "startMillis": 3703000, + "convergence": { + "nodes": 1, + "down": 0, + "needPlatformUpgrade": 0, + "upgrading": 0, + "needReboot": 0, + "rebooting": 0, + "needRestart": 0, + "restarting": 0, + "upgradingOs": 0, + "upgradingFirmware": 0, + "services": 1, + "needNewConfig": 1 + } + }, + "startStagingSetup": { + "status": "unfinished" + }, + "endStagingSetup": { + "status": "unfinished" + }, + "deployReal": { + "status": "unfinished" + }, + "installReal": { + "status": "unfinished" + }, + "startTests": { + "status": "unfinished" + }, + "endTests": { + "status": "unfinished" + }, + "copyVespaLogs": { + "status": "succeeded", + "startMillis": 3703000 + }, + "deactivateReal": { + "status": "succeeded", + "startMillis": 3703000 + }, + "deactivateTester": { + "status": "succeeded", + "startMillis": 3703000 + }, + "report": { + "status": "succeeded", + "startMillis": 3703000 + } + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 1f12f1e87af..9e7eeba8420 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -28,152 +28,57 @@ { "at": "(ignore)", "type": "info", - "message": "Checking installation of tester container ..." + "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated 6.1 " + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": "Wanted config generation is 2" + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: container on port 43 has config generation 1" + "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": "Installation of tester not yet complete." + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": "Checking installation of tester container ..." + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated 6.1 " + "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": "Wanted config generation is 2" + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: container on port 43 has config generation 1" + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "Installation of tester not yet complete." + "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": "Checking installation of tester container ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation of tester not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of tester container ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": "All services on wanted config generation." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Attempting to find deployment endpoints ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Endpoints not yet ready." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation of tester not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of tester container ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": "All services on wanted config generation." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Attempting to find deployment endpoints ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Found endpoints:" - }, - { - "at": "(ignore)", - "type": "info", - "message": "- test.us-east-1" - }, - { - "at": "(ignore)", - "type": "info", - "message": " |-- https://instance1-t--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": "--- platform 6.1" }, { "at": "(ignore)", @@ -207,177 +112,17 @@ { "at": "(ignore)", "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: container on port 43 has config generation 1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Installation not yet complete." - }, - { - "at": "(ignore)", - "type": "info", - "message": "Checking installation of 6.1 and 1.0.1-commit1 ..." - }, - { - "at": "(ignore)", - "type": "info", - "message": " host-tenant1:application1:instance1-test.us-east-1: unorchestrated 6.1 " - }, - { - "at": "(ignore)", - "type": "info", - "message": "Wanted config generation is 2" + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": "All services on wanted config generation." + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": "Attempting to find deployment endpoints ..." + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", @@ -449,7 +194,7 @@ } ] }, - "lastId": 85, + "lastId": 34, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json index 0fa4c541832..6c9315ca64b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json @@ -1,50 +1,27 @@ { + "active": false, + "status": "deploymentFailed", "log": { - "deployTester": [ + "deployReal": [ { "at": 1000, "type": "info", "message": "Failed to deploy application: ERROR!" } - ], - "deactivateTester": [ - { - "at": 1000, - "type": "info", - "message": "Deactivating tester of tenant.application in prod.us-east-3 ..." - } ] }, - "active": false, - "lastId": 2, + "lastId": 1, "steps": { - "startTests": { - "status": "unfinished" - }, - "deployTester": { - "startMillis": 1000, - "status": "failed" - }, - "report": { - "startMillis": 1000, - "status": "succeeded" - }, - "installTester": { - "status": "unfinished" - }, "deployReal": { - "status": "unfinished" + "status": "failed", + "startMillis": 1000 }, "installReal": { "status": "unfinished" }, - "deactivateTester": { - "startMillis": 1000, - "status": "succeeded" - }, - "endTests": { - "status": "unfinished" + "report": { + "status": "succeeded", + "startMillis": 1000 } - }, - "status": "deploymentFailed" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index b21c588235e..8029d650b75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -15,6 +18,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.Set; import static org.junit.Assert.assertFalse; @@ -156,4 +160,20 @@ public class ControllerApiTest extends ControllerContainerTest { tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json")); } + @Test + public void testMeteringApi() { + ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance"); + Instant timestamp = Instant.ofEpochMilli(123456789); + ZoneId zoneId = ZoneId.defaultId(); + List<ResourceSnapshot> snapshots = List.of( + new ResourceSnapshot(applicationId, 12,48,1200, timestamp, zoneId), + new ResourceSnapshot(applicationId, 24, 96,2400, timestamp, zoneId) + ); + tester.controller().serviceRegistry().meteringService().consume(snapshots); + tester.assertResponse( + operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET), + new File("metering.json") + ); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json new file mode 100644 index 00000000000..b64e8f26a63 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json @@ -0,0 +1,18 @@ +[ + { + "applicationId": "tenant.app.instance", + "timestamp": 123456789, + "zoneId": "prod.default", + "cpu": 12.0, + "memory": 48.0, + "disk": 1200.0 + }, + { + "applicationId": "tenant.app.instance", + "timestamp": 123456789, + "zoneId": "prod.default", + "cpu": 24.0, + "memory": 96.0, + "disk": 2400.0 + } +]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index d5c881bfe43..2579eede1ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -53,7 +53,8 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", - "failing": "staging-test" + "failing": "staging-test", + "status": "error" } ], "productionApplications": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index 82b97a5b144..bef27f7a2f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -73,13 +73,13 @@ public class AthenzRoleFilterTest { public void testTranslations() { // Hosted operators are always members of the hostedOperator role. - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH)); // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index a5520b42459..c95691fc120 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -43,6 +43,16 @@ public class ControllerAuthorizationFilterTest { } @Test + public void supporter() { + ControllerTester tester = new ControllerTester(); + SecurityContext securityContext = new SecurityContext(() -> "operator", Set.of(Role.hostedSupporter())); + ControllerAuthorizationFilter filter = createFilter(tester); + + assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/zone/v2/path", securityContext))); + assertIsAllowed(invokeFilter(filter, createRequest(Method.GET, "/zone/v1/path", securityContext))); + } + + @Test public void unprivileged() { ControllerTester tester = new ControllerTester(); SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.everyone())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json deleted file mode 100644 index dbaa6623fae..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "versions": [ - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud1", - "nodes": [ - { - "hostname": "node-2-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - } - ] - }, - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud2", - "nodes": [ - { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java new file mode 100644 index 00000000000..d191b460697 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -0,0 +1,298 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.routing; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +import static org.junit.Assert.assertNotEquals; + +/** + * @author mpolden + */ +public class RoutingApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/"; + + private ContainerTester tester; + private DeploymentTester deploymentTester; + + @Before + public void before() { + tester = new ContainerTester(container, responseFiles); + deploymentTester = new DeploymentTester(new ControllerTester(tester)); + } + + @Test + public void discovery() { + // Deploy + var context = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // GET root + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/", "", + Request.Method.GET), + new File("discovery/root.json")); + + // GET tenant + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", + Request.Method.GET), + new File("discovery/tenant.json")); + + // GET application + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/", + "", + Request.Method.GET), + new File("discovery/application.json")); + + // GET instance + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance.json")); + + // GET environment + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", + Request.Method.GET), + new File("discovery/environment.json")); + } + + @Test + public void recursion() { + var context1 = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var package1 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context1.submit(package1).deploy(); + + var context2 = deploymentTester.newDeploymentContext("t1", "a2", "default"); + var package2 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context2.submit(package2).deploy(); + + // GET tenant recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1?recursive=true", "", + Request.Method.GET), + new File("recursion/tenant.json")); + + // GET application recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET instance recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET environment recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment?recursive=true", "", + Request.Method.GET), + new File("recursion/environment.json")); + } + + @Test + public void exclusive_routing() { + var context = deploymentTester.newDeploymentContext(); + // Zones support direct routing + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + deploymentTester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(westZone), + ZoneApiMock.from(eastZone)); + // Deploy application + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + context.addRoutingPolicy(westZone, true); + context.addRoutingPolicy(eastZone, true); + + // GET initial deployment status + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/deployment-status-initial.json")); + + // POST sets deployment out + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.POST), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/deployment-status-out.json")); + + // DELETE sets deployment in + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.DELETE), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/deployment-status-in.json")); + + // GET initial zone status + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/zone-status-initial.json")); + + // POST sets zone out + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", + "", Request.Method.POST), + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/zone-status-out.json")); + + // DELETE sets zone in + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", + "", Request.Method.DELETE), + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("policy/zone-status-in.json")); + } + + @Test + public void shared_routing() { + // Deploy application + var context = deploymentTester.newDeploymentContext(); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + assertNotEquals("Rotation is assigned", List.of(), context.instance().rotations()); + + // GET initial deployment status + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/deployment-status-initial.json")); + + // POST sets deployment out + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.POST), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/deployment-status-out.json")); + + // DELETE sets deployment in + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.DELETE), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/deployment-status-in.json")); + + // GET initial zone status + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/zone-status-initial.json")); + + // POST sets zone out + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", + "", Request.Method.POST), + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/zone-status-out.json")); + + // DELETE sets zone in + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", + "", Request.Method.DELETE), + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("rotation/zone-status-in.json")); + } + + // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation + @Test + public void mixed_routing() { + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + + // One zone supports multiple routing methods + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), + RoutingMethod.shared, + RoutingMethod.exclusive); + + // Deploy application + var context = deploymentTester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // Assign policy in one zone + context.addRoutingPolicy(westZone, true); + + // GET status with both policy and rotation assigned + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("multi-status-initial.json")); + + // POST sets deployment out + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.POST), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("multi-status-out.json")); + + // DELETE sets deployment in + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.DELETE), + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + new File("multi-status-in.json")); + } + + @Test + public void invalid_requests() { + // GET non-existent application + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1", + "", Request.Method.GET), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"t1.a1 not found\"}", + 400); + + // GET non-existent zone + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-north-1", + "", Request.Method.GET), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such zone: prod.us-north-1\"}", + 400); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json new file mode 100644 index 00000000000..deda734cbbf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json new file mode 100644 index 00000000000..1e06b279873 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json @@ -0,0 +1,43 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/aws-us-east-2a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/us-east-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/perf/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-2/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-southeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-central-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/staging/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/test/region/us-east-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json new file mode 100644 index 00000000000..1a3ad823e14 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json new file mode 100644 index 00000000000..9b5630335aa --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json new file mode 100644 index 00000000000..acd05d35c8d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json new file mode 100644 index 00000000000..1c23c6bb569 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" + }, + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json new file mode 100644 index 00000000000..eea78c1b963 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "system", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json new file mode 100644 index 00000000000..6cb90bdb673 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" + }, + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json new file mode 100644 index 00000000000..59519c33d06 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json new file mode 100644 index 00000000000..e95d9bcdc42 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "system", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json new file mode 100644 index 00000000000..49b85775e63 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "exclusive", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json new file mode 100644 index 00000000000..abf0a46ae3e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "exclusive", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json new file mode 100644 index 00000000000..8328e1ffab1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "exclusive", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "system", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json new file mode 100644 index 00000000000..d86ca2d56e6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "exclusive", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json new file mode 100644 index 00000000000..e0b0e5e9b7a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json new file mode 100644 index 00000000000..f0dd0b7310d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json @@ -0,0 +1,108 @@ +{ + "zones": [ + { + "routingMethod": "shared", + "environment": "test", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "staging", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "aws-us-east-2a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "perf", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "aws-us-east-1a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-2", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-southeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-central-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "eu-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json new file mode 100644 index 00000000000..1ee4e1b82ba --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -0,0 +1,40 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json new file mode 100644 index 00000000000..5b15b72752c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json new file mode 100644 index 00000000000..90b2317c1b3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json new file mode 100644 index 00000000000..85e345c01d0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json @@ -0,0 +1,13 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "tenant:application:default", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json new file mode 100644 index 00000000000..eb06e9ee11d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json new file mode 100644 index 00000000000..eb06e9ee11d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json new file mode 100644 index 00000000000..440b80bc4d0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json @@ -0,0 +1,8 @@ +{ + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "out", + "agent": "operator", + "changedAt": "(ignore)" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 22ed079b501..1eb4523c344 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -111,8 +111,6 @@ public class SystemFlagsDeployerTest { SystemFlagsDeployResult result = deployer.deployFlags(archive, false); - System.out.println(result); - assertThat(result.errors()).containsOnly( OperationError.listFailed(exception.getMessage(), prodUsWest1Target)); assertThat(result.flagChanges()).containsOnly( @@ -123,4 +121,4 @@ public class SystemFlagsDeployerTest { return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename))); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d1dd50cfb4c..d70a09414bb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -203,7 +203,7 @@ public class UserApiTest extends ControllerContainerCloudTest { public void userMetadataTest() { ContainerTester tester = new ContainerTester(container, responseFiles); ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator()); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter()); User user = new User("dev@domail", "Joe Developer", "dev", null); tester.assertResponse(request("/api/user/v1/user") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 17489bb15d8..400fe8d4d9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -7,6 +7,7 @@ }, "tenants": {}, "operator": [ - "hostedOperator" + "hostedOperator", + "hostedSupporter" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 674f084a8b7..1bb531edbc5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -55,7 +55,7 @@ public class RotationRepositoryTest { @Before public void before() { tester = new DeploymentTester(new ControllerTester(rotationsConfig)); - repository = tester.applications().rotationRepository(); + repository = tester.controller().routingController().rotations(); application = tester.newDeploymentContext("tenant1", "app1", "default"); } @@ -83,7 +83,7 @@ public class RotationRepositoryTest { @Test public void strips_whitespace_in_rotation_fqdn() { tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); - RotationRepository repository = tester.controller().applications().rotationRepository(); + RotationRepository repository = tester.controller().routingController().rotations(); var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); application2.submit(applicationPackage); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java new file mode 100644 index 00000000000..a27b2e1084a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -0,0 +1,682 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.google.common.collect.Sets; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.Test; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author mortent + * @author mpolden + */ +public class RoutingPoliciesTest { + + private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); + private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); + private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); + + private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .build(); + + @Test + public void global_routing_policies() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); + int clustersPerZone = 2; + int numberOfDeployments = 2; + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0") + .endpoint("r1", "c0", "us-west-1") + .endpoint("r2", "c1") + .build(); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + + // Creates alias records + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); + assertEquals("Routing policy count is equal to cluster count", + numberOfDeployments * clustersPerZone, + tester.policiesOf(context1.instance().id()).size()); + + // Applications gains a new deployment + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .region(zone3.region()) + .endpoint("r0", "c0") + .endpoint("r1", "c0", "us-west-1") + .endpoint("r2", "c1") + .build(); + numberOfDeployments++; + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); + context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Endpoints are updated to contain cluster in new deployment + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); + + // Another application is deployed with a single cluster and global endpoint + var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; + tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); + var applicationPackage3 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0") + .build(); + context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // All endpoints for app1 are removed + ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .region(zone3.region()) + .allow(ValidationId.globalEndpointChange) + .build(); + context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); + var policies = tester.policiesOf(context1.instanceId()); + assertEquals(clustersPerZone * numberOfDeployments, policies.size()); + assertTrue("Rotation membership is removed from all policies", + policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); + assertEquals("Rotations for " + context2.application() + " are not removed", 2, tester.aliasDataOf(endpoint4).size()); + } + + @Test + public void zone_routing_policies() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); + + // Deploy application + int clustersPerZone = 2; + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Deployment creates records and policies for all clusters in all zones + Set<String> expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(4, tester.policiesOf(context1.instanceId()).size()); + + // Next deploy does nothing + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(4, tester.policiesOf(context1.instanceId()).size()); + + // Add 1 cluster in each zone and deploy + tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(6, tester.policiesOf(context1.instanceId()).size()); + + // Deploy another application + tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); + context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); + assertEquals(4, tester.policiesOf(context2.instanceId()).size()); + + // Deploy removes cluster from app1 + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + + // Remove app2 completely + tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() + .forEach(zone -> { + tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone); + tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone); + }); + context2.flushDnsUpdates(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty()); + assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size()); + } + + @Test + public void global_routing_policies_in_rotationless_system() { + var tester = new RoutingPoliciesTester(new DeploymentTester(new ControllerTester(new RotationsConfig.Builder().build()))); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region().value()) + .endpoint("r0", "c0") + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; + assertEquals(endpoint + " points to c0 in all regions", + List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), + tester.aliasDataOf(endpoint)); + assertTrue("No rotations assigned", context.application().instances().values().stream() + .map(Instance::rotations) + .allMatch(List::isEmpty)); + } + + @Test + public void cluster_endpoints_resolve_from_policies() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.provisionLoadBalancers(3, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), List.of()); + assertEquals(Map.of(ClusterSpec.Id.from("c0"), + URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), + ClusterSpec.Id.from("c1"), + URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), + ClusterSpec.Id.from("c2"), + URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), + tester.controllerTester().controller().routingController().zoneEndpointsOf(context.deploymentIdIn(zone1))); + } + + @Test + public void manual_deployment_creates_routing_policy() { + // Empty application package is valid in manually deployed environments + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + var emptyApplicationPackage = new ApplicationPackageBuilder().build(); + var zone = ZoneId.from("dev", "us-east-1"); + var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); + tester.controllerTester().serviceRegistry().zoneRegistry() + .setZones(zoneApi) + .exclusiveRoutingIn(zoneApi); + tester.provisionLoadBalancers(1, context.instanceId(), zone); + + // Deploy to dev + tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); + assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); + context.flushDnsUpdates(); + + // Routing policy is created and DNS is updated + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + assertEquals(Set.of("c0.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + } + + @Test + public void manual_deployment_creates_routing_policy_with_non_empty_spec() { + // Initial deployment + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + context.submit(applicationPackage).deploy(); + var zone = ZoneId.from("dev", "us-east-1"); + var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); + tester.controllerTester().serviceRegistry().zoneRegistry() + .setZones(zoneApi) + .exclusiveRoutingIn(zoneApi); + var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); + assertEquals(prodRecords, tester.recordNames()); + + // Deploy to dev under different instance + var devInstance = context.application().id().instance("user"); + tester.provisionLoadBalancers(1, devInstance, zone); + tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); + assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); + context.flushDnsUpdates(); + + // Routing policy is created and DNS is updated + assertEquals(1, tester.policiesOf(devInstance).size()); + assertEquals(Sets.union(prodRecords, Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); + } + + @Test + public void reprovisioning_load_balancer_preserves_cname_record() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Initial load balancer is provisioned + tester.provisionLoadBalancers(1, context.instanceId(), zone1); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .build(); + + // Application is deployed + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + var expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + + // Application is removed and the load balancer is deprovisioned + tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1); + tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); + + // Load balancer for the same application is provisioned again, but with a different hostname + var newHostname = HostName.from("new-hostname"); + var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), + context.instanceId(), + ClusterSpec.Id.from("c0"), + newHostname, + LoadBalancer.State.active, + Optional.of("dns-zone-1")); + tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); + + // Application redeployment preserves DNS record + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + assertEquals("CNAME points to current load blancer", newHostname.value() + ".", + tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); + } + + @Test + public void set_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + // Global routing status is overridden in one zone + var changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + + // Inactive zone is removed from global DNS record + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Status details is stored in policy + var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Other zone remains in + var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); + assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); + + // Next deployment does not affect status + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Deployment is set back in + tester.controllerTester().clock().advance(Duration.ofHours(1)); + changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Deployment is set out through a new deployment.xml + var applicationPackage2 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region(), false) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); + + // ... back in + var applicationPackage3 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + } + + @Test + public void set_zone_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); + var contexts = List.of(context1, context2); + + // Deploy applications + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) + .build(); + for (var context : contexts) { + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + // Set zone out + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + for (var context : contexts) { + var policies = tester.routingPolicies().get(context.instanceId()); + assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, + policies.values().stream() + .map(RoutingPolicy::status) + .map(Status::globalRouting) + .map(GlobalRouting::status) + .allMatch(status -> status == GlobalRouting.Status.in)); + } + var changedAt = tester.controllerTester().clock().instant(); + var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); + assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); + assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); + + // Setting status per deployment does not affect status as entire zone is out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + + // Set single deployment out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + + // Set zone back in. Deployment set explicitly out, remains out, the rest are in + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + @Test + public void non_production_deployment_is_not_registered_in_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + + // Configure the system to use the same region for test, staging and prod + var sharedRegion = RegionName.from("aws-us-east-1c"); + var prodZone = ZoneId.from(Environment.prod, sharedRegion); + var stagingZone = ZoneId.from(Environment.staging, sharedRegion); + var testZone = ZoneId.from(Environment.test, sharedRegion); + var zones = List.of(ZoneApiMock.from(prodZone), + ZoneApiMock.from(stagingZone), + ZoneApiMock.from(testZone)); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), + SystemApplication.all()); + + var context = tester.tester.newDeploymentContext(); + var endpointId = EndpointId.of("r0"); + var applicationPackage = new ApplicationPackageBuilder() + .region(sharedRegion) + .endpoint(endpointId.id(), "default") + .build(); + + // Application starts deployment + context = context.submit(applicationPackage); + for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + context = context.runJob(testJob); + // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the + // deployment, we consume only one DNS update at a time here + do { + context = context.flushDnsUpdates(1); + tester.assertTargets(context.instanceId(), endpointId, 0); + } while (!tester.recordNames().isEmpty()); + } + + // Deployment completes + context.completeRollout(); + tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); + } + + @Test + public void changing_global_routing_status_never_removes_all_members() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created, pointing to all configured zones + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Global routing status is overridden for one deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + + // Setting other deployment out implicitly sets all deployments in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // One inactive deployment is put back in. Global DNS record now points to the only active deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Setting zone (containing active deployment) out puts all deployments in + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); + context.flushDnsUpdates(); + assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Setting zone back in removes the currently inactive deployment + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Inactive deployment is set in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { + assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); + } + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + } + + private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { + List<LoadBalancer> loadBalancers = new ArrayList<>(); + for (int i = 0; i < count; i++) { + loadBalancers.add( + new LoadBalancer("LB-" + i + "-Z-" + zone.value(), + application, + ClusterSpec.Id.from("c" + i), + HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()), + LoadBalancer.State.active, + Optional.of("dns-zone-1"))); + } + return loadBalancers; + } + + private static class RoutingPoliciesTester { + + private final DeploymentTester tester; + + public RoutingPoliciesTester() { + this(SystemName.main); + } + + public RoutingPoliciesTester(SystemName system) { + this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system)))); + } + + public RoutingPolicies routingPolicies() { + return tester.controllerTester().controller().routingController().policies(); + } + + public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { + return tester.newDeploymentContext(tenant, application, instance); + } + + public ControllerTester controllerTester() { + return tester.controllerTester(); + } + + public RoutingPoliciesTester(DeploymentTester tester) { + this.tester = tester; + // Make all zones directly routed + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); + } + + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + for (ZoneId zone : zones) { + tester.configServer().removeLoadBalancers(application, zone); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); + } + } + + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { + return tester.controller().curator().readRoutingPolicies(instance).values(); + } + + private Set<String> recordNames() { + return tester.controllerTester().nameService().records().stream() + .map(Record::name) + .map(RecordName::asString) + .collect(Collectors.toSet()); + } + + private List<String> aliasDataOf(String name) { + return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() + .map(Record::data) + .map(RecordData::asString) + .collect(Collectors.toList()); + } + + private List<String> cnameDataOf(String name) { + return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() + .map(Record::data) + .map(RecordData::asString) + .collect(Collectors.toList()); + } + + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { + var endpoint = RoutingPolicy.globalEndpointOf(application, endpointId, tester.controller().system()).dnsName(); + var zoneTargets = Arrays.stream(zone) + .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + + z.value() + "/dns-zone-1/" + z.value()) + .collect(Collectors.toSet()); + assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, + Set.copyOf(aliasDataOf(endpoint))); + } + + } + +} diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index 235ca7cb698..125c5004d25 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -15,49 +15,6 @@ <binding>http://*/tester/v1/*</binding> </handler> - <http> - <!-- Make sure 4080 is the first port. This will be used by the config server. --> - <server id='default' port='4080'/> - <server id='testertls4443' port='4443'> - <config name="jdisc.http.connector"> - <tlsClientAuthEnforcer> - <enable>true</enable> - <pathWhitelist> - <item>/status.html</item> - <item>/state/v1/config</item> - </pathWhitelist> - </tlsClientAuthEnforcer> - </config> - <ssl> - <private-key-file>/var/lib/sia/keys/vespa.vespa.tenant.key.pem</private-key-file> - <certificate-file>/var/lib/sia/certs/vespa.vespa.tenant.cert.pem</certificate-file> - <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file> - <client-authentication>want</client-authentication> - </ssl> - </server> - <filtering> - <access-control domain='vespa.vespa.cd'> - <exclude> - <binding>http://*/tester/v1/*</binding> - </exclude> - </access-control> - <request-chain id="testrunner-api"> - <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle="jdisc-security-filters"> - <config name="jdisc.http.filter.security.athenz.athenz-authorization-filter"> - <credentialsToVerify>TOKEN_ONLY</credentialsToVerify> - <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName> - </config> - <component id="com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper" bundle="jdisc-security-filters"> - <config name="jdisc.http.filter.security.athenz.static-request-resource-mapper"> - <resourceName>vespa.vespa.cd:tester-application</resourceName> - <action>deploy</action> - </config> - </component> - </filter> - </request-chain> - </filtering> - </http> - <nodes count="1" allocated-memory="17%"> <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> </nodes> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index a99a391e81a..e29e4c32017 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -4,12 +4,13 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_rhel_6_10) message("-- Setting up default build settings for rhel 6.10") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_rhel_7_7) message("-- Setting up default build settings for rhel 7.7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) endfunction() @@ -21,13 +22,14 @@ endfunction() function(setup_vespa_default_build_settings_centos_7) message("-- Setting up default build settings for centos 7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_darwin) @@ -39,14 +41,14 @@ function(setup_vespa_default_build_settings_darwin) else() set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endif() - set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" PARENT_SCOPE) - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib") + set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" PARENT_SCOPE) + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY) list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}") endif() list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include") if(DEFINED DEFAULT_LLVM_INCLUDE_DIRECTORY) list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_LLVM_INCLUDE_DIRECTORY}") endif() @@ -56,22 +58,26 @@ endfunction() function(setup_vespa_default_build_settings_fedora_29) message("-- Setting up default build settings for fedora 29") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_30) message("-- Setting up default build settings for fedora 30") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_31) message("-- Setting up default build settings for fedora 31") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_32) message("-- Setting up default build settings for fedora 32") - set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_ubuntu_18_10) @@ -81,9 +87,45 @@ function(setup_vespa_default_build_settings_ubuntu_18_10) set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE) endfunction() +function(vespa_use_default_vespa_unprivileged) + if(NOT DEFINED VESPA_UNPRIVILEGED) + message("-- Setting VESPA_UNPRIVILEGED to yes") + set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE) + endif() +endfunction() + +function(vespa_use_default_cmake_install_prefix) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa") + if(COMMAND vespa_use_specific_install_prefix) + vespa_use_specific_install_prefix() + endif() + else() + set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa") + endif() + message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") + set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE) + endif() +endfunction() + +function(vespa_use_default_vespa_user) + if(NOT DEFINED VESPA_USER) + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_VESPA_USER "vespa") + if(COMMAND vespa_use_specific_vespa_user) + vespa_use_specific_vespa_user() + endif() + else() + set(DEFAULT_VESPA_USER "$ENV{USER}") + endif() + message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}") + set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE) + endif() +endfunction() + function(vespa_use_default_build_settings) - if (DEFINED CMAKE_INSTALL_PREFIX AND DEFINED CMAKE_PREFIX_PATH AND - NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND + if (DEFINED CMAKE_PREFIX_PATH AND DEFINED VESPA_LLVM_VERSION AND DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH) @@ -94,26 +136,7 @@ function(vespa_use_default_build_settings) unset(DEFAULT_CMAKE_PREFIX_PATH) unset(DEFAULT_EXTRA_LINK_DIRECTORY) unset(DEFAULT_EXTRA_INCLUDE_DIRECTORY) - unset(DEFAULT_VESPA_USER) unset(DEFAULT_VESPA_CPU_ARCH_FLAGS) - if(NOT DEFINED VESPA_UNPRIVILEGED) - message("-- Setting VESPA_UNPRIVILEGED to yes") - set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE) - set(VESPA_UNPRIVILEGED "yes") - endif() - if(VESPA_UNPRIVILEGED STREQUAL "no") - set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa") - set(DEFAULT_VESPA_USER "vespa") - if(COMMAND vespa_use_specific_install_prefix) - vespa_use_specific_install_prefix() - endif() - if(COMMAND vespa_use_specific_vespa_user) - vespa_use_specific_vespa_user() - endif() - else() - set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa") - set(DEFAULT_VESPA_USER "$ENV{USER}") - endif() if(APPLE) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(VESPA_DEPS "/opt/vespa-deps-clang") @@ -169,12 +192,11 @@ function(vespa_use_default_build_settings) endif() if(NOT DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS) if(VESPA_OS_DISTRO STREQUAL "fedora" AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=westmere") + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=westmere -mtune=haswell") + else() + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-mtune=intel") endif() endif() - if(DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) - message("-- DEFAULT_CMAKE_INSTALL_PREFIX is ${DEFAULT_CMAKE_INSTALL_PREFIX}") - endif() if(DEFINED DEFAULT_CMAKE_PREFIX_PATH) message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") endif() @@ -187,16 +209,7 @@ function(vespa_use_default_build_settings) if(DEFINED DEFAULT_VESPA_LLVM_VERSION) message("-- DEFAULT_VESPA_LLVM_VERSION is ${DEFAULT_VESPA_LLVM_VERSION}") endif() - if(DEFINED DEFAULT_VESPA_USER) - message("-- DEFAULT_VESPA_USER is ${DEFAULT_VESPA_USER}") - endif() - if(DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS) - message("-- DEFAULT_VESPA_CPU_ARCH_FLAGS is ${DEFAULT_VESPA_CPU_ARCH_FLAGS}") - endif() - if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) - message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") - set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE) - endif() + message("-- DEFAULT_VESPA_CPU_ARCH_FLAGS is ${DEFAULT_VESPA_CPU_ARCH_FLAGS}") if(NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED DEFAULT_CMAKE_PREFIX_PATH) message("-- Setting CMAKE_PREFIX_PATH to ${DEFAULT_CMAKE_PREFIX_PATH}") set(CMAKE_PREFIX_PATH "${DEFAULT_CMAKE_PREFIX_PATH}" PARENT_SCOPE) @@ -237,10 +250,6 @@ function(vespa_use_default_build_settings) message("-- Setting VESPA_LLVM_VERSION to ${DEFAULT_VESPA_LLVM_VERSION}") set(VESPA_LLVM_VERSION "${DEFAULT_VESPA_LLVM_VERSION}" PARENT_SCOPE) endif() - if(NOT DEFINED VESPA_USER AND DEFINED DEFAULT_VESPA_USER) - message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}") - set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE) - endif() if(NOT DEFINED VESPA_CPU_ARCH_FLAGS AND DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS) message("-- Setting VESPA_CPU_ARCH_FLAGS to ${DEFAULT_VESPA_CPU_ARCH_FLAGS}") set(VESPA_CPU_ARCH_FLAGS "${DEFAULT_VESPA_CPU_ARCH_FLAGS}" PARENT_SCOPE) diff --git a/dist/vespa.spec b/dist/vespa.spec index 9daeac2a1c1..c54e4442167 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -47,10 +47,11 @@ BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: vespa-openssl-devel >= 1.1.1c-1 +BuildRequires: vespa-icu-devel >= 65.1.0-1 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 -BuildRequires: llvm-devel >= 7.0.1-3 +BuildRequires: llvm-devel >= 8.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: openssl-devel BuildRequires: vespa-gtest >= 1.8.1-1 @@ -80,17 +81,20 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} -BuildRequires: llvm-devel >= 9.0.0 +BuildRequires: llvm-devel >= 10.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %endif BuildRequires: xxhash-devel >= 0.6.5 +BuildRequires: openblas-devel BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel +%if ! 0%{?el7} BuildRequires: libicu-devel +%endif BuildRequires: java-11-openjdk-devel BuildRequires: rpm-build BuildRequires: make @@ -116,37 +120,43 @@ Requires: perl-IO-Socket-IP Requires: perl-JSON Requires: perl-libwww-perl Requires: perl-LWP-Protocol-https -%if !(0%{?centos} && 0%{?el8}) Requires: perl-Net-INET6Glue -%endif Requires: perl-Pod-Usage Requires: perl-URI Requires: valgrind Requires: Judy Requires: xxhash Requires: xxhash-libs >= 0.6.5 +%if 0%{?el8} +Requires: openblas +%else +Requires: openblas-serial +%endif Requires: lz4 Requires: libzstd Requires: zlib +%if ! 0%{?el7} Requires: libicu +%endif Requires: perf Requires: gdb Requires: net-tools %if 0%{?el7} Requires: llvm5.0 Requires: vespa-openssl >= 1.1.1c-1 +Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 %define _vespa_llvm_version 5.0 %define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64 -%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include +%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?el8} -Requires: llvm-libs >= 7.0.1-3 +Requires: llvm-libs >= 8.0.1 Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs -%define _vespa_llvm_version 7.0 +%define _vespa_llvm_version 8 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 -%define _extra_include_directory %{_vespa_deps_prefix}/include +%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} Requires: vespa-protobuf >= 3.7.0-4 @@ -164,11 +174,11 @@ Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %if 0%{?fc32} -Requires: llvm-libs >= 9.0.0 -%define _vespa_llvm_version 9 +Requires: llvm-libs >= 10.0.0 +%define _vespa_llvm_version 10 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 -%define _extra_include_directory %{_vespa_deps_prefix}/include +%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif Requires: java-11-openjdk Requires(pre): shadow-utils diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java index ffd5dffece9..c7b9d342043 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java @@ -124,4 +124,8 @@ public class ContainerResources { public ContainerResources withMemoryBytes(long memoryBytes) { return new ContainerResources(cpus, cpuShares, memoryBytes); } + + public ContainerResources withUnlimitedCpus() { + return new ContainerResources(0, 0, memoryBytes); + } } diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java index 6868bc8ecd4..4160f366fdb 100644 --- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -37,8 +37,8 @@ public class ProcessingUpdateTestCase { @Test public void testProcessingUpdates() { DocumentType articleType = new DocumentType("article"); - Field bodyField = new Field("body", DataType.STRING, true); - Field titleField = new Field("title", DataType.STRING, true); + Field bodyField = new Field("body", DataType.STRING); + Field titleField = new Field("title", DataType.STRING); articleType.addField(bodyField); articleType.addField(titleField); dtm = new DocumentTypeManager(); diff --git a/document/abi-spec.json b/document/abi-spec.json index 6ce2543b4c2..abbf03bc228 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -427,6 +427,8 @@ "methods": [ "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)", + "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.document.DocumentType clone()", "public com.yahoo.document.Document createFieldValue()", "public java.lang.Class getValueClass()", @@ -448,6 +450,8 @@ "public com.yahoo.document.Field getField(int)", "public boolean hasField(java.lang.String)", "public int getFieldCount()", + "public java.util.Set getImportedFieldNames()", + "public boolean hasImportedField(java.lang.String)", "public com.yahoo.document.Field removeField(java.lang.String)", "public java.util.Collection getFields()", "public java.util.Set fieldSet()", @@ -712,6 +716,8 @@ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)", "public com.yahoo.document.DocumenttypesConfig$Documenttype build()" ], "fields": [ @@ -719,7 +725,8 @@ "public java.util.List datatype", "public java.util.List annotationtype", "public java.util.Map fieldsets", - "public java.util.List referencetype" + "public java.util.List referencetype", + "public java.util.List importedfield" ] }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": { @@ -1264,6 +1271,35 @@ ], "fields": [] }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()" + ], + "fields": [] + }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public java.lang.String name()" + ], + "fields": [] + }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -1347,7 +1383,9 @@ "public java.util.Map fieldsets()", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)", "public java.util.List referencetype()", - "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)" + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)", + "public java.util.List importedfield()", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)" ], "fields": [] }, @@ -1457,8 +1495,10 @@ ], "methods": [ "public void <init>(java.lang.String, int, com.yahoo.document.DataType, boolean)", + "public void <init>(java.lang.String, int, com.yahoo.document.DataType)", "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean, com.yahoo.document.DocumentType)", + "public void <init>(java.lang.String, com.yahoo.document.DataType, com.yahoo.document.DocumentType)", "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean)", "public void <init>(java.lang.String, com.yahoo.document.DataType)", "public void <init>(java.lang.String, com.yahoo.document.Field)", @@ -3083,6 +3123,7 @@ "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Integer)", "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Long)", "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Byte)", + "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Boolean)", "public abstract com.yahoo.document.datatypes.FieldValue removeFieldValue(com.yahoo.document.Field)", "public com.yahoo.document.datatypes.FieldValue removeFieldValue(java.lang.String)", "public abstract void clear()", diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index f3ff6e4ed30..23559878fbb 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType { private StructDataType bodyType; private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); + private final Set<String> importedFieldNames; /** * Creates a new document type and registers it with the document type manager. @@ -51,8 +52,7 @@ public class DocumentType extends StructuredDataType { * @param name The name of the new document type */ public DocumentType(String name) { - this(name, new StructDataType(name + ".header"), - new StructDataType(name + ".body")); + this(name, createHeaderStructType(name), createBodyStructType(name)); } /** @@ -65,9 +65,27 @@ public class DocumentType extends StructuredDataType { * @param bodyType The type of the body struct */ public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { + this(name, headerType, bodyType, Collections.emptySet()); + } + + public DocumentType(String name, StructDataType headerType, + StructDataType bodyType, Set<String> importedFieldNames) { super(name); this.headerType = headerType; this.bodyType = bodyType; + this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); + } + + public DocumentType(String name, Set<String> importedFieldNames) { + this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames); + } + + private static StructDataType createHeaderStructType(String name) { + return new StructDataType(name + ".header"); + } + + private static StructDataType createBodyStructType(String name) { + return new StructDataType(name + ".body"); } @Override @@ -181,8 +199,7 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } - StructDataType struct = (field.isHeader() ? headerType : bodyType); - struct.addField(field); + headerType.addField(field); } // Do not use, public only for testing @@ -221,8 +238,8 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } - Field field = new Field(name, type, false); - bodyType.addField(field); + Field field = new Field(name, type); + headerType.addField(field); return field; } @@ -234,13 +251,9 @@ public class DocumentType extends StructuredDataType { * @return The field created * TODO Fix searchdefinition so that exception can be thrown if filed is already registerd */ + @Deprecated public Field addHeaderField(String name, DataType type) { - if (isRegistered()) { - throw new IllegalStateException("You cannot add fields to a document type that is already registered."); - } - Field field = new Field(name, type, true); - headerType.addField(field); - return field; + return addField(name, type); } /** @@ -374,6 +387,14 @@ public class DocumentType extends StructuredDataType { return headerType.getFieldCount() + bodyType.getFieldCount(); } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + + public boolean hasImportedField(String fieldName) { + return importedFieldNames.contains(fieldName); + } + /** * Removes an field from the DocumentType. * diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 154c25880a9..21a163aa2b9 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -10,8 +10,10 @@ import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Configures the Vespa document manager from a config id. @@ -144,7 +146,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (Field field : body.getFields()) { field.setHeader(false); } - DocumentType type = new DocumentType(doc.name(), header, body); + var importedFields = doc.importedfield().stream() + .map(f -> f.name()) + .collect(Collectors.toUnmodifiableSet()); + DocumentType type = new DocumentType(doc.name(), header, body, importedFields); for (Object j : doc.inherits()) { DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; @@ -180,9 +185,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub : manager.getDataType(field.datatype(), field.detailedtype()); if (field.id().size() == 1) { - type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); + type.addField(new Field(field.name(), field.id().get(0).id(), fieldType)); } else { - type.addField(new Field(field.name(), fieldType, true)); + type.addField(new Field(field.name(), fieldType)); } } manager.register(type); diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 671c8c7f763..9be4036174c 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -21,7 +21,6 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab protected DataType dataType; protected int fieldId; - private boolean isHeader; private boolean forcedId; /** @@ -32,11 +31,14 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param isHeader Whether this is a "header" field or a "content" field * (true = "header"). */ + @Deprecated public Field(String name, int id, DataType dataType, boolean isHeader) { + this(name, id, dataType); + } + public Field(String name, int id, DataType dataType) { super(name); this.fieldId = id; this.dataType = dataType; - this.isHeader = isHeader; this.forcedId = true; validateId(id, null); } @@ -55,8 +57,13 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * (true = "header"). * @param owner the owning document (used to check for id collisions) */ + @Deprecated public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) { - this(name, 0, dataType, isHeader); + this(name, dataType, owner); + } + + public Field(String name, DataType dataType, DocumentType owner) { + this(name, 0, dataType); this.fieldId = calculateIdV7(owner); this.forcedId = false; } @@ -69,8 +76,9 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param isHeader Whether this is a "header" field or a "content" field * (true = "header"). */ + @Deprecated public Field(String name, DataType dataType, boolean isHeader) { - this(name, dataType, isHeader, null); + this(name, dataType); } /** @@ -80,7 +88,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param dataType The datatype of the field */ public Field(String name, DataType dataType) { - this(name, dataType, true); + this(name, dataType, null); } /** @@ -89,7 +97,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab */ // TODO: Decide on one copy/clone idiom and do it for this and all it is calling public Field(String name, Field field) { - this(name, field.dataType, field.isHeader, null); + this(name, field.dataType, null); } public int compareTo(Object o) { @@ -196,14 +204,12 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ @Deprecated // TODO: Remove on Vespa 8 public boolean isHeader() { - return isHeader; + return true; } /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ @Deprecated // TODO: Remove on Vespa 8 - public void setHeader(boolean header) { - this.isHeader = header; - } + public void setHeader(boolean header) { } /** Two fields are equal if they have the same name and the same data type */ @Override diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java index 295b1c9dd5c..134681a4028 100755 --- a/document/src/main/java/com/yahoo/document/FieldPath.java +++ b/document/src/main/java/com/yahoo/document/FieldPath.java @@ -10,7 +10,7 @@ import java.util.List; * This class represents a path into a document, that can be used to iterate through the document and extract the field * values you're interested in. * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public class FieldPath implements Iterable<FieldPathEntry> { diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java index b3ea93d8467..43016187954 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -139,6 +139,10 @@ public abstract class StructuredFieldValue extends CompositeFieldValue { public final FieldValue setFieldValue(String field, Byte value) { return setFieldValue(field, new ByteFieldValue(value)); } + + public final FieldValue setFieldValue(String field, Boolean value) { + return setFieldValue(field, new BoolFieldValue(value)); + } /** * Removes and returns a field value. * diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index 0cedda7c4f0..9e2759e1590 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -7,6 +7,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentGet; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.FieldPath; import com.yahoo.document.datatypes.FieldPathIteratorHandler; @@ -131,10 +132,37 @@ public class AttributeNode implements ExpressionNode { throw new IllegalStateException("Function '" + function + "' is not supported."); } - private static Object evaluateFieldPath(String fieldPth, Object value) { + private static boolean looksLikeComplexFieldPath(String path) { + for (int i = 0; i < path.length(); ++i) { + switch (path.charAt(i)) { + case '.': + case '{': + case '[': + return true; + } + } + return false; + } + + private static boolean isSimpleImportedField(String path, DocumentType documentType) { + if (looksLikeComplexFieldPath(path)) { + return false; + } + return documentType.hasImportedField(path); + } + + private static Object evaluateFieldPath(String fieldPathStr, Object value) { if (value instanceof DocumentPut) { final Document doc = ((DocumentPut) value).getDocument(); - FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth); + if (isSimpleImportedField(fieldPathStr, doc.getDataType())) { + // Imported fields can only be meaningfully evaluated in the backend, so we + // explicitly treat them as if they are valid fields with missing values. This + // will be treated the same as if it's a normal field by the selection operators. + // This avoids any awkward interaction with Invalid values or having to + // augment the FieldPath code with knowledge of imported fields. + return null; + } + FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPathStr); IteratorHandler handler = new IteratorHandler(); doc.iterateNested(fieldPath, 0, handler); if (handler.values.isEmpty()) { diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index 47599e53ece..8f52c29e84d 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -144,9 +144,9 @@ public class ComparisonNode implements ExpressionNode { return new ResultList(Result.INVALID); } } else if (oLeft instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight); + return evaluateLhsListAndRhsSingle((AttributeNode.VariableValueList)oLeft, oRight); } else if (oRight instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft); + return evaluateLhsSingleAndRhsList(oLeft, (AttributeNode.VariableValueList)oRight); } return new ResultList(evaluateBool(oLeft, oRight)); } @@ -197,7 +197,7 @@ public class ComparisonNode implements ExpressionNode { } } - private ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) { + private ResultList evaluateLhsListAndRhsSingle(AttributeNode.VariableValueList lhs, Object rhs) { if (rhs == null && lhs == null) { return new ResultList(Result.TRUE); } @@ -215,6 +215,24 @@ public class ComparisonNode implements ExpressionNode { return retVal; } + private ResultList evaluateLhsSingleAndRhsList(Object lhs, AttributeNode.VariableValueList rhs) { + if (rhs == null && lhs == null) { + return new ResultList(Result.TRUE); + } + + if (rhs == null || lhs == null) { + return new ResultList(Result.FALSE); + } + + ResultList retVal = new ResultList(); + for (ResultList.VariableValue value : rhs) { + Result result = evaluateBool(lhs, value.getValue()); + retVal.add((FieldPathIteratorHandler.VariableMap)value.getVariables().clone(), result); + } + + return retVal; + } + /** * Evaluate this expression on two operands, given that they are not invalid. * diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index c2111edfd10..630f204c44d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -82,7 +82,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } public void write(Document doc) { - write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); + write(new Field(doc.getDataType().getName(), 0, doc.getDataType()), doc); } @SuppressWarnings("deprecation") diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 0d6b0cae926..5db98f26141 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -320,7 +320,7 @@ public final class XmlDocumentWriter implements DocumentWriter { buffer = new XmlStream(); buffer.setIndent(indent); optionalWrapperMarker.clear(); - write(new Field(document.getDataType().getName(), 0, document.getDataType(), true), document); + write(new Field(document.getDataType().getName(), 0, document.getDataType()), document); } @Override diff --git a/document/src/test/document/documentmanager.importedfields.cfg b/document/src/test/document/documentmanager.importedfields.cfg new file mode 100644 index 00000000000..765e290ae82 --- /dev/null +++ b/document/src/test/document/documentmanager.importedfields.cfg @@ -0,0 +1,65 @@ +datatype[7] +datatype[0].id -1365874599 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name referenced_type.header +datatype[0].structtype[0].version 9 +datatype[0].structtype[0].field[0] +datatype[0].documenttype[0] +datatype[1].id 278604398 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name referenced_type.body +datatype[1].structtype[0].version 9 +datatype[1].structtype[0].field[0] +datatype[1].documenttype[0] +datatype[2].id 124647170 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name referenced_type +datatype[2].documenttype[0].version 9 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct -1365874599 +datatype[2].documenttype[0].bodystruct 278604398 +datatype[3].id 12345678 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[3].referencetype[1] +datatype[3].referencetype[0].target_type_id 124647170 +datatype[4].id 673066331 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name type_with_ref.header +datatype[4].structtype[0].version 234 +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name my_ref_field +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 12345678 +datatype[4].documenttype[0] +datatype[5].id -176986064 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[1] +datatype[5].structtype[0].name type_with_ref.body +datatype[5].structtype[0].version 234 +datatype[5].structtype[0].field[0] +datatype[5].documenttype[0] +datatype[6].id -1293964543 +datatype[6].arraytype[0] +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[1] +datatype[6].documenttype[0].name type_with_ref +datatype[6].documenttype[0].version 234 +datatype[6].documenttype[0].inherits[0] +datatype[6].documenttype[0].headerstruct 673066331 +datatype[6].documenttype[0].bodystruct -176986064 +datatype[6].documenttype[0].importedfield[0].name "my_cool_imported_field" +datatype[6].documenttype[0].importedfield[1].name "my_awesome_imported_field"
\ No newline at end of file diff --git a/document/src/test/document/serializecpp-lz4-level9.dat b/document/src/test/document/serializecpp-lz4-level9.dat Binary files differindex db759871107..1dffaa2d7a7 100644 --- a/document/src/test/document/serializecpp-lz4-level9.dat +++ b/document/src/test/document/serializecpp-lz4-level9.dat diff --git a/document/src/test/document/serializecpp.dat b/document/src/test/document/serializecpp.dat Binary files differindex 6435dc088ac..d004d11e47b 100644 --- a/document/src/test/document/serializecpp.dat +++ b/document/src/test/document/serializecpp.dat diff --git a/document/src/test/document/serializecppsplit_header.dat b/document/src/test/document/serializecppsplit_header.dat Binary files differindex 08176e4e737..d004d11e47b 100755 --- a/document/src/test/document/serializecppsplit_header.dat +++ b/document/src/test/document/serializecppsplit_header.dat diff --git a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java index fb2d478d38b..36cc18ebd6b 100755 --- a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java @@ -27,11 +27,11 @@ public class DocumentCalculatorTestCase { docMan = new DocumentTypeManager(); testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("byteattr", DataType.BYTE); - testDocType.addHeaderField("intattr", DataType.INT); - testDocType.addHeaderField("longattr", DataType.LONG); - testDocType.addHeaderField("doubleattr", DataType.DOUBLE); - testDocType.addHeaderField("missingattr", DataType.INT); + testDocType.addField("byteattr", DataType.BYTE); + testDocType.addField("intattr", DataType.INT); + testDocType.addField("longattr", DataType.LONG); + testDocType.addField("doubleattr", DataType.DOUBLE); + testDocType.addField("missingattr", DataType.INT); docMan.registerDocumentType(testDocType); doc = new Document(testDocType, new DocumentId("id:ns:testdoc::testdoc:http://www.ntnu.no/")); diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java index 08abd6e6a2d..fea3b265b6d 100644 --- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java @@ -34,11 +34,11 @@ public class DocumentIdTestCase { public void setUp() { DocumentType testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("intattr", DataType.INT); + testDocType.addField("intattr", DataType.INT); testDocType.addField("rawattr", DataType.RAW); testDocType.addField("floatattr", DataType.FLOAT); - testDocType.addHeaderField("stringattr", DataType.STRING); - testDocType.addHeaderField("Minattr", DataType.INT); + testDocType.addField("stringattr", DataType.STRING); + testDocType.addField("Minattr", DataType.INT); manager.registerDocumentType(testDocType); } diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index bc1224ca8ea..fa47c80c6fb 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -62,25 +62,25 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { public void testSerializationAllVersions() throws IOException { DocumentType docInDocType = new DocumentType("docindoc"); - docInDocType.addField(new Field("stringindocfield", DataType.STRING, false)); + docInDocType.addField(new Field("stringindocfield", DataType.STRING)); DocumentType docType = new DocumentType("serializetest"); - docType.addField(new Field("floatfield", DataType.FLOAT, true)); - docType.addField(new Field("stringfield", DataType.STRING, true)); - docType.addField(new Field("longfield", DataType.LONG, true)); - docType.addField(new Field("urifield", DataType.URI, true)); - docType.addField(new Field("intfield", DataType.INT, false)); - docType.addField(new Field("rawfield", DataType.RAW, false)); - docType.addField(new Field("doublefield", DataType.DOUBLE, false)); - docType.addField(new Field("bytefield", DataType.BYTE, false)); - docType.addField(new Field("boolfield", DataType.BOOL, false)); + docType.addField(new Field("floatfield", DataType.FLOAT)); + docType.addField(new Field("stringfield", DataType.STRING)); + docType.addField(new Field("longfield", DataType.LONG)); + docType.addField(new Field("urifield", DataType.URI)); + docType.addField(new Field("intfield", DataType.INT)); + docType.addField(new Field("rawfield", DataType.RAW)); + docType.addField(new Field("doublefield", DataType.DOUBLE)); + docType.addField(new Field("bytefield", DataType.BYTE)); + docType.addField(new Field("boolfield", DataType.BOOL)); DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT); - docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false)); + docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType)); DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType); - docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false)); - docType.addField(new Field("docfield", DataType.DOCUMENT, false)); + docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType)); + docType.addField(new Field("docfield", DataType.DOCUMENT)); DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false); - docType.addField(new Field("wsfield", weightedSetDataType, false)); + docType.addField(new Field("wsfield", weightedSetDataType)); DocumentTypeManager docMan = new DocumentTypeManager(); docMan.register(docInDocType); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 141a74a24fe..dcd4622b3f4 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -107,26 +107,26 @@ public class DocumentTestCase extends DocumentTestCaseBase { docMan = new DocumentTypeManager(); DocumentType docInDocType = new DocumentType("docindoc"); - docInDocType.addField(new Field("tull", 2, docMan.getDataType(2), true)); + docInDocType.addField(new Field("tull", 2, docMan.getDataType(2))); docMan.registerDocumentType(docInDocType); DocumentType sertestDocType = new DocumentType("sertest"); - sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("date", 3, docMan.getDataType(0), true)); - sertestDocType.addField(new Field("from", 4, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("to", 6, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("body", 10, docMan.getDataType(2), false)); - sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0), false)); - sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)), false)); - sertestDocType.addField(new Field("rawfield", 879, DataType.RAW, false)); - sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING), false)); - sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false), false)); - sertestDocType.addField(new Field("docindoc", 882, docInDocType, false)); - sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false)); - sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false)); - sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL, false)); + sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2))); + sertestDocType.addField(new Field("date", 3, docMan.getDataType(0))); + sertestDocType.addField(new Field("from", 4, docMan.getDataType(2))); + sertestDocType.addField(new Field("to", 6, docMan.getDataType(2))); + sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2))); + sertestDocType.addField(new Field("body", 10, docMan.getDataType(2))); + sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0))); + sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)))); + sertestDocType.addField(new Field("rawfield", 879, DataType.RAW)); + sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING))); + sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false))); + sertestDocType.addField(new Field("docindoc", 882, docInDocType)); + sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING))); + sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE)); + sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL)); docMan.registerDocumentType(sertestDocType); } @@ -880,13 +880,13 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testInheritance() { // Create types that inherit each other.. And test that it works.. DocumentType parentType = new DocumentType("parent"); - parentType.addField(new Field("parentbodyint", DataType.INT, false)); - parentType.addField(new Field("parentheaderint", DataType.INT, true)); - parentType.addField(new Field("overwritten", DataType.INT, true)); + parentType.addField(new Field("parentbodyint", DataType.INT)); + parentType.addField(new Field("parentheaderint", DataType.INT)); + parentType.addField(new Field("overwritten", DataType.INT)); DocumentType childType = new DocumentType("child"); - childType.addField(new Field("childbodyint", DataType.INT, false)); - childType.addField(new Field("childheaderint", DataType.INT, true)); - childType.addField(new Field("overwritten", DataType.INT, true)); + childType.addField(new Field("childbodyint", DataType.INT)); + childType.addField(new Field("childheaderint", DataType.INT)); + childType.addField(new Field("overwritten", DataType.INT)); childType.inherit(parentType); DocumentTypeManager manager = new DocumentTypeManager(); @@ -914,13 +914,13 @@ public class DocumentTestCase extends DocumentTestCaseBase { @Test public void testInheritanceTypeMismatch() { DocumentType parentType = new DocumentType("parent"); - parentType.addField(new Field("parentbodyint", DataType.INT, false)); - parentType.addField(new Field("parentheaderint", DataType.INT, true)); - parentType.addField(new Field("overwritten", DataType.STRING, true)); + parentType.addField(new Field("parentbodyint", DataType.INT)); + parentType.addField(new Field("parentheaderint", DataType.INT)); + parentType.addField(new Field("overwritten", DataType.STRING)); DocumentType childType = new DocumentType("child"); - childType.addField(new Field("childbodyint", DataType.INT, false)); - childType.addField(new Field("childheaderint", DataType.INT, true)); - childType.addField(new Field("overwritten", DataType.INT, true)); + childType.addField(new Field("childbodyint", DataType.INT)); + childType.addField(new Field("childheaderint", DataType.INT)); + childType.addField(new Field("overwritten", DataType.INT)); try { childType.inherit(parentType); fail("Inheritance with conflicting types worked."); @@ -934,7 +934,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testFieldValueImplementations() { docMan = new DocumentTypeManager(); DocumentType docType = new DocumentType("impl"); - docType.addField(new Field("something", DataType.getArray(DataType.STRING), false)); + docType.addField(new Field("something", DataType.getArray(DataType.STRING))); docMan.register(docType); //just checks that isAssignableFrom() in Document.setFieldValue() goes the right way @@ -1276,9 +1276,9 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testDocumentComparisonDoesNotCorruptStateBug6394548() { DocumentTypeManager docMan = new DocumentTypeManager(); DocumentType docType = new DocumentType("bug2354045"); - docType.addField(new Field("string", 2, DataType.STRING, true)); - docType.addField(new Field("int", 1, DataType.INT, true)); - docType.addField(new Field("float", 0, DataType.FLOAT, true)); + docType.addField(new Field("string", 2, DataType.STRING)); + docType.addField(new Field("int", 1, DataType.INT)); + docType.addField(new Field("float", 0, DataType.FLOAT)); docMan.register(docType); Document doc1 = new Document(docType, new DocumentId("id:ns:bug2354045::bug6394548")); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java index 68fe1c8cc57..6f95f77f08c 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java @@ -23,14 +23,14 @@ public class DocumentTestCaseBase { docMan = new DocumentTypeManager(); testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("byteattr", DataType.BYTE); - testDocType.addHeaderField("intattr", DataType.INT); + testDocType.addField("byteattr", DataType.BYTE); + testDocType.addField("intattr", DataType.INT); testDocType.addField("rawattr", DataType.RAW); testDocType.addField("floatattr", DataType.FLOAT); - testDocType.addHeaderField("stringattr", DataType.STRING); - testDocType.addHeaderField("Minattr", DataType.INT); - testDocType.addHeaderField("Minattr2", DataType.INT); - testDocType.addHeaderField("primitive1", DataType.INT); + testDocType.addField("stringattr", DataType.STRING); + testDocType.addField("Minattr", DataType.INT); + testDocType.addField("Minattr2", DataType.INT); + testDocType.addField("primitive1", DataType.INT); StructDataType sdt = new StructDataType("struct1"); sdt.addField(new Field("primitive1", DataType.INT)); diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index 65c217e09e1..57b36d9758f 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.StructuredFieldValue; import org.junit.Test; +import java.util.HashSet; import java.util.Iterator; import static org.junit.Assert.assertEquals; @@ -30,7 +31,7 @@ public class DocumentTypeManagerTestCase { DocumentType newDocType = new DocumentType("testdoc"); newDocType.addField("Fjomp", DataType.INT); - newDocType.addHeaderField("Fjols", DataType.STRING); + newDocType.addField("Fjols", DataType.STRING); manager.registerDocumentType(newDocType); @@ -40,7 +41,6 @@ public class DocumentTypeManagerTestCase { assertEquals("Fjomp", fetched4.getName()); assertEquals(fetched4.getDataType(), DataType.INT); - assertEquals(fetched4.isHeader(), false); } @Test @@ -64,8 +64,8 @@ public class DocumentTypeManagerTestCase { StructDataType struct = new StructDataType("mystruct"); DataType wset1 = DataType.getWeightedSet(DataType.getArray(DataType.INT)); DataType wset2 = DataType.getWeightedSet(DataType.getArray(DataType.TAG)); - struct.addField(new Field("foo", wset1, true)); - struct.addField(new Field("bar", wset2, false)); + struct.addField(new Field("foo", wset1)); + struct.addField(new Field("bar", wset2)); DataType array = DataType.getArray(struct); DocumentType docType = new DocumentType("mydoc"); docType.addField("hmm", array); @@ -148,7 +148,6 @@ public class DocumentTypeManagerTestCase { assertTrue(type.hasField("foobarfield1")); Field foobarfield0 = type.getField("foobarfield0"); - assertTrue(!foobarfield0.isHeader()); assertTrue(foobarfield0.getDataType().getCode() == 2); Field foobarfield1 = type.getField("foobarfield1"); @@ -190,7 +189,6 @@ public class DocumentTypeManagerTestCase { assertTrue(type.hasField("arrayarrayfloat")); Field arrayfloat = type.getField("arrayfloat"); - assertTrue(!arrayfloat.isHeader()); ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType(); assertTrue(dataType.getCode() == 99); assertTrue(dataType.getValueClass().equals(Array.class)); @@ -200,7 +198,6 @@ public class DocumentTypeManagerTestCase { Field arrayarrayfloat = type.getField("arrayarrayfloat"); ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType(); - assertTrue(!arrayarrayfloat.isHeader()); assertTrue(subType.getCode() == 4003); assertTrue(subType.getValueClass().equals(Array.class)); assertTrue(subType.getNestedType().getCode() == 99); @@ -218,14 +215,14 @@ public class DocumentTypeManagerTestCase { DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes")); assertNull(banana.getField("newfield")); - assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99), false), customtypes.getField("arrayfloat")); + assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99)), customtypes.getField("arrayfloat")); DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg"); banana = manager.getDocumentType(new DataTypeName("banana")); customtypes = manager.getDocumentType(new DataTypeName("customtypes")); - assertEquals(new Field("newfield", 12345, DataType.STRING, true), banana.getField("newfield")); + assertEquals(new Field("newfield", 12345, DataType.STRING), banana.getField("newfield")); assertNull(customtypes.getField("arrayfloat")); } @@ -559,6 +556,35 @@ search annotationsimplicitstruct { assertTrue(fieldRefType.getTargetType() == targetDocType); } + @Test + public void imported_fields_are_empty_if_no_fields_provided_in_config() { + var manager = createConfiguredManager("file:src/test/document/documentmanager.singlereference.cfg"); + var docType = manager.getDocumentType("type_with_ref"); + + assertNotNull(docType.getImportedFieldNames()); + assertEquals(docType.getImportedFieldNames().size(), 0); + assertFalse(docType.hasImportedField("foo")); + } + + @Test + public void imported_fields_are_populated_from_config() { + var manager = createConfiguredManager("file:src/test/document/documentmanager.importedfields.cfg"); + var docType = manager.getDocumentType("type_with_ref"); + + var expectedFields = new HashSet<String>(); + expectedFields.add("my_cool_imported_field"); + expectedFields.add("my_awesome_imported_field"); + assertEquals(docType.getImportedFieldNames(), expectedFields); + + assertTrue(docType.hasImportedField("my_cool_imported_field")); + assertTrue(docType.hasImportedField("my_awesome_imported_field")); + assertFalse(docType.hasImportedField("a_missing_imported_field")); + } + + // TODO test clone(). Also fieldSets not part of clone()..! + + // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this... + // TODO test reference to own doc type } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index c381a093b4e..9f05a4441b2 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -799,8 +799,8 @@ public class DocumentUpdateTestCase { public TensorUpdateSerializeFixture() { docMan = new DocumentTypeManager(); docType = new DocumentType("test"); - docType.addHeaderField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})"))); - docType.addHeaderField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])"))); + docType.addField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})"))); + docType.addField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])"))); docMan.registerDocumentType(docType); } diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java index 286f7d72b24..7cf0adf6ed1 100644 --- a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java +++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java @@ -17,9 +17,9 @@ public class IncompatibleFieldTypesTest { public void setUp() { arrayOfStrings = new ArrayDataType(DataType.STRING); struct = new StructDataType("fancypants"); - struct.addField(new Field("stringarray", arrayOfStrings, false)); + struct.addField(new Field("stringarray", arrayOfStrings)); DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false); - struct.addField(new Field("stringws", weightedSetOfStrings, false)); + struct.addField(new Field("stringws", weightedSetOfStrings)); root = struct.createFieldValue(); root.setFieldValue("stringarray", arrayOfStrings.createFieldValue()); diff --git a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java index 4197938f0ee..295cfa37e89 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java @@ -20,14 +20,14 @@ public class StructTestCase { @Test public void testBasicStuff() throws Exception { StructDataType type = new StructDataType("teststr"); - type.addField(new Field("int", 0, DataType.INT, true)); - type.addField(new Field("flt", 1, DataType.FLOAT, true)); - type.addField(new Field("str", 2, DataType.STRING, true)); - type.addField(new Field("raw", 3, DataType.RAW, true)); - type.addField(new Field("lng", 4, DataType.LONG, true)); - type.addField(new Field("dbl", 5, DataType.DOUBLE, true)); - type.addField(new Field("uri", 6, DataType.URI, true)); - type.addField(new Field("byt", 8, DataType.BYTE, true)); + type.addField(new Field("int", 0, DataType.INT)); + type.addField(new Field("flt", 1, DataType.FLOAT)); + type.addField(new Field("str", 2, DataType.STRING)); + type.addField(new Field("raw", 3, DataType.RAW)); + type.addField(new Field("lng", 4, DataType.LONG)); + type.addField(new Field("dbl", 5, DataType.DOUBLE)); + type.addField(new Field("uri", 6, DataType.URI)); + type.addField(new Field("byt", 8, DataType.BYTE)); Struct struct = new Struct(type); { @@ -236,7 +236,7 @@ public class StructTestCase { @Test public void testSetUnknownType() { StructDataType type = new StructDataType("teststr"); - type.addField(new Field("int", 0, DataType.INT, true)); + type.addField(new Field("int", 0, DataType.INT)); Struct struct = new Struct(type); try { @@ -251,9 +251,9 @@ public class StructTestCase { public void testCompareToDoesNotMutateStateBug6394548() { StructDataType type = new StructDataType("test"); // NOTE: non-increasing ID order! - type.addField(new Field("int", 2, DataType.INT, true)); - type.addField(new Field("flt", 1, DataType.FLOAT, true)); - type.addField(new Field("str", 0, DataType.STRING, true)); + type.addField(new Field("int", 2, DataType.INT)); + type.addField(new Field("flt", 1, DataType.FLOAT)); + type.addField(new Field("str", 0, DataType.STRING)); Struct a = new Struct(type); a.setFieldValue("int", new IntegerFieldValue(123)); diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java index c11c58ff729..404b069277b 100644 --- a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java +++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java @@ -98,13 +98,13 @@ public class FieldSetTestCase extends DocumentTestCaseBase { assertFalse(new DocIdOnly().contains(headerField)); assertTrue(new HeaderFields().contains(headerField)); - assertFalse(new HeaderFields().contains(bodyField)); + assertTrue(new HeaderFields().contains(bodyField)); assertTrue(new HeaderFields().contains(new DocIdOnly())); assertTrue(new HeaderFields().contains(new NoFields())); - assertContains("[body]", "testdoc:rawattr"); + assertNotContains("[body]", "testdoc:rawattr"); assertContains("[header]", "testdoc:intattr"); - assertNotContains("[header]", "testdoc:rawattr"); + assertContains("[header]", "testdoc:rawattr"); assertContains("testdoc:rawattr,intattr", "testdoc:intattr"); assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr"); assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr"); @@ -141,10 +141,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase { Document doc = getTestDocument(); doc.removeFieldValue("rawattr"); - assertEquals("floatattr:3.56", doCopyFields(doc, "[body]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doCopyFields(doc, "[all]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]")); - assertEquals("byteattr:30,floatattr:3.56", doCopyFields(doc, "testdoc:floatattr,byteattr")); + assertEquals("", doCopyFields(doc, "[body]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[all]")); + assertEquals("floatattr:3.56,byteattr:30", doCopyFields(doc, "testdoc:floatattr,byteattr")); } String doStripFields(Document source, String fieldSet) { @@ -159,10 +159,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase { Document doc = getTestDocument(); doc.removeFieldValue("rawattr"); - assertEquals("floatattr:3.56", doStripFields(doc, "[body]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doStripFields(doc, "[all]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]")); - assertEquals("byteattr:30,floatattr:3.56", doStripFields(doc, "testdoc:floatattr,byteattr")); + assertEquals("", doStripFields(doc, "[body]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[all]")); + assertEquals("floatattr:3.56,byteattr:30", doStripFields(doc, "testdoc:floatattr,byteattr")); } @Test diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 0ff0bd81d90..4e591cdfbd4 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -8,10 +8,13 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.yolean.Exceptions; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -26,20 +29,24 @@ import static org.junit.Assert.fail; */ public class DocumentSelectorTestCase { + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + private static DocumentTypeManager manager = new DocumentTypeManager(); @Before public void setUp() { - DocumentType type = new DocumentType("test"); - type.addHeaderField("hint", DataType.INT); - type.addHeaderField("hfloat", DataType.FLOAT); - type.addHeaderField("hstring", DataType.STRING); + var importedFields = new HashSet<>(List.of("my_imported_field")); + DocumentType type = new DocumentType("test", importedFields); + type.addField("hint", DataType.INT); + type.addField("hfloat", DataType.FLOAT); + type.addField("hstring", DataType.STRING); type.addField("content", DataType.STRING); StructDataType mystruct = new StructDataType("mystruct"); - mystruct.addField(new Field("key", DataType.INT, false)); - mystruct.addField(new Field("value", DataType.STRING, false)); - type.addHeaderField("mystruct", mystruct); + mystruct.addField(new Field("key", DataType.INT)); + mystruct.addField(new Field("value", DataType.STRING)); + type.addField("mystruct", mystruct); ArrayDataType structarray = new ArrayDataType(mystruct); type.addField("structarray", structarray); @@ -697,6 +704,44 @@ public class DocumentSelectorTestCase { } @Test + public void using_non_commutative_comparison_operator_with_field_value_is_well_defined() throws ParseException { + var documents = createDocs(); + // Doc 0 contains 24 in `hint` field. + assertEquals(Result.FALSE, evaluate("25 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("25 > test.hint", documents.get(0))); + assertEquals(Result.FALSE, evaluate("24 > test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 >= test.hint", documents.get(0))); + + assertEquals(Result.FALSE, evaluate("test.hint <= 23", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint <= 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint > 23", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.hint > 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint >= 24", documents.get(0))); + } + + @Test + public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException { + var documents = createDocs(); + assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.my_imported_field != null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.my_imported_field", documents.get(0))); + // Only (in)equality operators are well defined for null values; everything else becomes Invalid. + assertEquals(Result.INVALID, evaluate("test.my_imported_field > 0", documents.get(0))); + } + + @Test + public void imported_fields_only_supported_for_simple_expressions() throws ParseException { + exceptionRule.expect(IllegalArgumentException.class); + // TODO we should probably handle this case specially and give a better exception message + exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test"); + + var documents = createDocs(); + // Nested field access is NOT considered a simple expression. + evaluate("test.my_imported_field.foo", documents.get(0)); + } + + @Test public void testTicket1769674() { assertParseError("music.uri=\"junk", "Lexical error at line -1, column 17. Encountered: <EOF> after : \"\\\"junk\""); diff --git a/document/src/test/resources/predicates/false__cpp b/document/src/test/resources/predicates/false__cpp Binary files differindex 00a71d5fe73..c9f426b3aaf 100644 --- a/document/src/test/resources/predicates/false__cpp +++ b/document/src/test/resources/predicates/false__cpp diff --git a/document/src/test/resources/predicates/false__java b/document/src/test/resources/predicates/false__java Binary files differindex 00a71d5fe73..c9f426b3aaf 100644 --- a/document/src/test/resources/predicates/false__java +++ b/document/src/test/resources/predicates/false__java diff --git a/document/src/test/resources/predicates/foo_in_6_9__cpp b/document/src/test/resources/predicates/foo_in_6_9__cpp Binary files differindex f9f7249e65b..e063a3aad6a 100644 --- a/document/src/test/resources/predicates/foo_in_6_9__cpp +++ b/document/src/test/resources/predicates/foo_in_6_9__cpp diff --git a/document/src/test/resources/predicates/foo_in_6_9__java b/document/src/test/resources/predicates/foo_in_6_9__java Binary files differindex d18b937bead..9c1413f1b77 100644 --- a/document/src/test/resources/predicates/foo_in_6_9__java +++ b/document/src/test/resources/predicates/foo_in_6_9__java diff --git a/document/src/test/resources/predicates/foo_in_6_x__cpp b/document/src/test/resources/predicates/foo_in_6_x__cpp Binary files differindex 0ccf8e9794b..e2b261668ed 100644 --- a/document/src/test/resources/predicates/foo_in_6_x__cpp +++ b/document/src/test/resources/predicates/foo_in_6_x__cpp diff --git a/document/src/test/resources/predicates/foo_in_6_x__java b/document/src/test/resources/predicates/foo_in_6_x__java Binary files differindex b8ae334882e..d53f4d03996 100644 --- a/document/src/test/resources/predicates/foo_in_6_x__java +++ b/document/src/test/resources/predicates/foo_in_6_x__java diff --git a/document/src/test/resources/predicates/foo_in_bar__cpp b/document/src/test/resources/predicates/foo_in_bar__cpp Binary files differindex a2761427c6a..8a19ba3dc58 100644 --- a/document/src/test/resources/predicates/foo_in_bar__cpp +++ b/document/src/test/resources/predicates/foo_in_bar__cpp diff --git a/document/src/test/resources/predicates/foo_in_bar__java b/document/src/test/resources/predicates/foo_in_bar__java Binary files differindex aeb10e2b5d7..3adde3ae2fe 100644 --- a/document/src/test/resources/predicates/foo_in_bar__java +++ b/document/src/test/resources/predicates/foo_in_bar__java diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp Binary files differindex 8a2705e6f62..8583f37da57 100644 --- a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp +++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__cpp diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java Binary files differindex 8a2705e6f62..8583f37da57 100644 --- a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java +++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__cpp b/document/src/test/resources/predicates/foo_in_bar_baz__cpp Binary files differindex b4cf08bbf57..ed5bc82e2eb 100644 --- a/document/src/test/resources/predicates/foo_in_bar_baz__cpp +++ b/document/src/test/resources/predicates/foo_in_bar_baz__cpp diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__java b/document/src/test/resources/predicates/foo_in_bar_baz__java Binary files differindex ea3314d9bd7..1db1a175c0c 100644 --- a/document/src/test/resources/predicates/foo_in_bar_baz__java +++ b/document/src/test/resources/predicates/foo_in_bar_baz__java diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp Binary files differindex 4d5474c24e4..3b1dbd541fa 100644 --- a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp +++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__cpp diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java Binary files differindex 4d5474c24e4..3b1dbd541fa 100644 --- a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java +++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java diff --git a/document/src/test/resources/predicates/foo_in_x_9__cpp b/document/src/test/resources/predicates/foo_in_x_9__cpp Binary files differindex 29218b9e944..9490703c980 100644 --- a/document/src/test/resources/predicates/foo_in_x_9__cpp +++ b/document/src/test/resources/predicates/foo_in_x_9__cpp diff --git a/document/src/test/resources/predicates/foo_in_x_9__java b/document/src/test/resources/predicates/foo_in_x_9__java Binary files differindex 017a610a7d5..9a2bfc9af89 100644 --- a/document/src/test/resources/predicates/foo_in_x_9__java +++ b/document/src/test/resources/predicates/foo_in_x_9__java diff --git a/document/src/test/resources/predicates/foo_in_x__cpp b/document/src/test/resources/predicates/foo_in_x__cpp Binary files differindex bd2916835c6..636fdffb856 100644 --- a/document/src/test/resources/predicates/foo_in_x__cpp +++ b/document/src/test/resources/predicates/foo_in_x__cpp diff --git a/document/src/test/resources/predicates/foo_in_x__java b/document/src/test/resources/predicates/foo_in_x__java Binary files differindex 6537cc6bdeb..8faf7762be7 100644 --- a/document/src/test/resources/predicates/foo_in_x__java +++ b/document/src/test/resources/predicates/foo_in_x__java diff --git a/document/src/test/resources/predicates/foo_in_x_x__cpp b/document/src/test/resources/predicates/foo_in_x_x__cpp Binary files differindex e0d8282c46d..57e88bc4e80 100644 --- a/document/src/test/resources/predicates/foo_in_x_x__cpp +++ b/document/src/test/resources/predicates/foo_in_x_x__cpp diff --git a/document/src/test/resources/predicates/foo_in_x_x__java b/document/src/test/resources/predicates/foo_in_x_x__java Binary files differindex 5060718417a..bf01a265651 100644 --- a/document/src/test/resources/predicates/foo_in_x_x__java +++ b/document/src/test/resources/predicates/foo_in_x_x__java diff --git a/document/src/test/resources/predicates/not_foo_in_bar__cpp b/document/src/test/resources/predicates/not_foo_in_bar__cpp Binary files differindex b654de6d53e..a7cbacbaf35 100644 --- a/document/src/test/resources/predicates/not_foo_in_bar__cpp +++ b/document/src/test/resources/predicates/not_foo_in_bar__cpp diff --git a/document/src/test/resources/predicates/not_foo_in_bar__java b/document/src/test/resources/predicates/not_foo_in_bar__java Binary files differindex b654de6d53e..a7cbacbaf35 100644 --- a/document/src/test/resources/predicates/not_foo_in_bar__java +++ b/document/src/test/resources/predicates/not_foo_in_bar__java diff --git a/document/src/test/resources/predicates/true__cpp b/document/src/test/resources/predicates/true__cpp Binary files differindex 2b5da7409d5..87356a5ed44 100644 --- a/document/src/test/resources/predicates/true__cpp +++ b/document/src/test/resources/predicates/true__cpp diff --git a/document/src/test/resources/predicates/true__java b/document/src/test/resources/predicates/true__java Binary files differindex 2b5da7409d5..87356a5ed44 100644 --- a/document/src/test/resources/predicates/true__java +++ b/document/src/test/resources/predicates/true__java diff --git a/document/src/test/resources/tensor/empty_tensor__cpp b/document/src/test/resources/tensor/empty_tensor__cpp Binary files differindex 2c15c152558..cf878f0e689 100644 --- a/document/src/test/resources/tensor/empty_tensor__cpp +++ b/document/src/test/resources/tensor/empty_tensor__cpp diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java Binary files differindex 2c15c152558..cf878f0e689 100644 --- a/document/src/test/resources/tensor/empty_tensor__java +++ b/document/src/test/resources/tensor/empty_tensor__java diff --git a/document/src/test/resources/tensor/multi_cell_tensor__cpp b/document/src/test/resources/tensor/multi_cell_tensor__cpp Binary files differindex d4c7c5fbbe5..9adda236a4a 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__cpp +++ b/document/src/test/resources/tensor/multi_cell_tensor__cpp diff --git a/document/src/test/resources/tensor/multi_cell_tensor__java b/document/src/test/resources/tensor/multi_cell_tensor__java Binary files differindex d923fc10559..deb53463fb5 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__java +++ b/document/src/test/resources/tensor/multi_cell_tensor__java diff --git a/document/src/test/resources/tensor/non_existing_tensor__cpp b/document/src/test/resources/tensor/non_existing_tensor__cpp Binary files differindex 08cbcac6dd3..7a1d95ff132 100644 --- a/document/src/test/resources/tensor/non_existing_tensor__cpp +++ b/document/src/test/resources/tensor/non_existing_tensor__cpp diff --git a/document/src/test/resources/tensor/non_existing_tensor__java b/document/src/test/resources/tensor/non_existing_tensor__java Binary files differindex 08cbcac6dd3..7a1d95ff132 100644 --- a/document/src/test/resources/tensor/non_existing_tensor__java +++ b/document/src/test/resources/tensor/non_existing_tensor__java diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index 014884e44cd..0bdf51194fb 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -15,9 +15,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -53,40 +52,39 @@ TEST(ArrayFieldValueTest, testArray) EXPECT_EQ(IntFieldValue(3), (IntFieldValue&) value[2]); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream stream(value.serialize()); ArrayFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(stream, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By index - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.remove(1); EXPECT_TRUE(!value2.contains(IntFieldValue(2))); EXPECT_EQ(size_t(2), value2.size()); // By value - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // By value with multiple present - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.add(IntFieldValue(1)); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/tests/data/document-cpp-currentversion-lz4-9.dat b/document/src/tests/data/document-cpp-currentversion-lz4-9.dat Binary files differindex 7b0650996db..3383d97f253 100644 --- a/document/src/tests/data/document-cpp-currentversion-lz4-9.dat +++ b/document/src/tests/data/document-cpp-currentversion-lz4-9.dat diff --git a/document/src/tests/data/document-cpp-currentversion-uncompressed.dat b/document/src/tests/data/document-cpp-currentversion-uncompressed.dat Binary files differindex 2b39c42f8a4..07d63edf576 100644 --- a/document/src/tests/data/document-cpp-currentversion-uncompressed.dat +++ b/document/src/tests/data/document-cpp-currentversion-uncompressed.dat diff --git a/document/src/tests/data/serializejava-compressed.dat b/document/src/tests/data/serializejava-compressed.dat Binary files differindex 0ac391422ff..e11bb3b53db 100644 --- a/document/src/tests/data/serializejava-compressed.dat +++ b/document/src/tests/data/serializejava-compressed.dat diff --git a/document/src/tests/data/serializejava.dat b/document/src/tests/data/serializejava.dat Binary files differindex 10873c9d905..3fa21bdccd3 100644 --- a/document/src/tests/data/serializejava.dat +++ b/document/src/tests/data/serializejava.dat diff --git a/document/src/tests/data/serializejavawithannotations.dat b/document/src/tests/data/serializejavawithannotations.dat Binary files differindex fe683d1580e..08854d03a29 100644 --- a/document/src/tests/data/serializejavawithannotations.dat +++ b/document/src/tests/data/serializejavawithannotations.dat diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 79b849c5ba9..6d446f6f1d7 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -16,6 +16,7 @@ #include <vespa/document/select/invalidconstant.h> #include <vespa/document/select/doctype.h> #include <vespa/document/select/compare.h> +#include <vespa/document/select/operator.h> #include <vespa/document/select/parse_utils.h> #include <vespa/vespalib/util/exceptions.h> #include <limits> @@ -56,16 +57,6 @@ protected: void SetUp() override; void createDocs(); - void testOperators0(); - void testOperators1(); - void testOperators2(); - void testOperators3(); - void testOperators4(); - void testOperators5(); - void testOperators6(); - void testOperators7(); - void testOperators8(); - void testOperators9(); void testDocumentUpdates0(); void testDocumentUpdates1(); void testDocumentUpdates2(); @@ -92,6 +83,10 @@ void DocumentSelectParserTest::SetUp() builder.document(-1673092522, "usergroup", Struct("usergroup.header"), Struct("usergroup.body")); + builder.document(1234567, "with_imported", + Struct("with_imported.header"), + Struct("with_imported.body")) + .imported_field("my_imported_field"); _repo = std::make_unique<DocumentTypeRepo>(builder.config()); _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); @@ -103,7 +98,7 @@ Document::SP DocumentSelectParserTest::createDoc( uint64_t hlong) { const DocumentType* type = _repo->getDocumentType(doctype); - Document::SP doc(new Document(*type, DocumentId(id))); + auto doc = std::make_shared<Document>(*type, DocumentId(id)); doc->setValue(doc->getField("headerval"), IntFieldValue(hint)); if (hlong != 0) { @@ -498,21 +493,7 @@ DocumentSelectParserTest::doParse(vespalib::stringref expr, EXPECT_EQ(select::ResultList(select::Result::result), \ doParse(expr, (doc).getId())) << (std::string("Doc id: ") + expr); -TEST_F(DocumentSelectParserTest, testOperators) -{ - testOperators0(); - testOperators1(); - testOperators2(); - testOperators3(); - testOperators4(); - testOperators5(); - testOperators6(); - testOperators7(); - testOperators8(); - testOperators9(); -} - -void DocumentSelectParserTest::testOperators0() +TEST_F(DocumentSelectParserTest, operators_0) { createDocs(); @@ -551,8 +532,17 @@ void DocumentSelectParserTest::testOperators0() PARSE("\"foo\" == 'foo'", *_doc[0], True); PARSE("\"bar\" = \"a\"", *_doc[0], False); PARSE("\"bar\" = \"*a*\"", *_doc[0], True); + PARSE("\"bar\" = \"*x*\"", *_doc[0], False); + PARSE("\"bar\" = \"ba*\"", *_doc[0], True); + PARSE("\"bar\" = \"a*\"", *_doc[0], False) + PARSE("\"bar\" = \"*ar\"", *_doc[0], True); + PARSE("\"bar\" = \"*a\"", *_doc[0], False); PARSE("\"bar\" = \"\"", *_doc[0], False); PARSE("\"\" = \"\"", *_doc[0], True); + PARSE("\"\" = \"*\"", *_doc[0], True); + PARSE("\"\" = \"****\"", *_doc[0], True); + PARSE("\"a\" = \"*?*\"", *_doc[0], True); + PARSE("\"a\" = \"*??*\"", *_doc[0], False); PARSE("\"bar\" =~ \"^a$\"", *_doc[0], False); PARSE("\"bar\" =~ \"a\"", *_doc[0], True); PARSE("\"bar\" =~ \"\"", *_doc[0], True); @@ -561,7 +551,32 @@ void DocumentSelectParserTest::testOperators0() PARSE("30 = 30", *_doc[0], True); } -void DocumentSelectParserTest::testOperators1() +TEST_F(DocumentSelectParserTest, using_non_commutative_comparison_operator_with_field_value_is_well_defined) { + auto doc = createDoc("testdoctype1", "id:foo:testdoctype1::bar", 24, 0.0, "foo", "bar", 0); + // Document's `headerval` field has value of 24. + PARSE("25 <= testdoctype1.headerval", *doc, False); + PARSE("24 <= testdoctype1.headerval", *doc, True); + PARSE("25 > testdoctype1.headerval", *doc, True); + PARSE("24 > testdoctype1.headerval", *doc, False); + PARSE("24 >= testdoctype1.headerval", *doc, True); + + PARSE("testdoctype1.headerval <= 23", *doc, False); + PARSE("testdoctype1.headerval <= 24", *doc, True); + PARSE("testdoctype1.headerval > 23", *doc, True); + PARSE("testdoctype1.headerval > 24", *doc, False); + PARSE("testdoctype1.headerval >= 24", *doc, True); +} + +TEST_F(DocumentSelectParserTest, regex_matching_does_not_bind_anchors_to_newlines) { + createDocs(); + + PARSE("\"a\\nb\\nc\" =~ \"^b$\"", *_doc[0], False); + PARSE("\"a\\r\\nb\\r\\nc\" =~ \"^b$\"", *_doc[0], False); + // Same applies to implicit regex created from glob expression + PARSE("\"a\\nb\\nc\" = \"b\"", *_doc[0], False); +} + +TEST_F(DocumentSelectParserTest, operators_1) { createDocs(); @@ -608,7 +623,7 @@ void DocumentSelectParserTest::testOperators1() PARSE("testdoctype1.headerval = 10", *_doc[4], True); } -void DocumentSelectParserTest::testOperators2() +TEST_F(DocumentSelectParserTest, operators_2) { createDocs(); @@ -633,7 +648,7 @@ void DocumentSelectParserTest::testOperators2() PARSEI("id.group == \"xyzzy\"", *_doc[10], True); } -void DocumentSelectParserTest::testOperators3() +TEST_F(DocumentSelectParserTest, operators_3) { createDocs(); { @@ -666,7 +681,7 @@ void DocumentSelectParserTest::testOperators3() PARSEI("id == \"id:footype:testdoctype1:n=123456789:badger\"", *_doc[5], False); } -void DocumentSelectParserTest::testOperators4() +TEST_F(DocumentSelectParserTest, operators_4) { createDocs(); @@ -693,7 +708,7 @@ void DocumentSelectParserTest::testOperators4() PARSE("false or testdoctype1.content = 1", *_doc[0], Invalid); } -void DocumentSelectParserTest::testOperators5() +TEST_F(DocumentSelectParserTest, operators_5) { createDocs(); @@ -722,7 +737,7 @@ void DocumentSelectParserTest::testOperators5() PARSEI("-6 % 10 = -6", *_doc[0], True); } -void DocumentSelectParserTest::testOperators6() +TEST_F(DocumentSelectParserTest, operators_6) { createDocs(); @@ -766,7 +781,7 @@ void DocumentSelectParserTest::testOperators6() PARSE("testdoctype1.headerlongval<0", *_doc[7], True); } -void DocumentSelectParserTest::testOperators7() +TEST_F(DocumentSelectParserTest, operators_7) { createDocs(); @@ -803,7 +818,7 @@ void DocumentSelectParserTest::testOperators7() PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$y].value == \"structval2\"", *_doc[1], True); } -void DocumentSelectParserTest::testOperators8() +TEST_F(DocumentSelectParserTest, operators_8) { createDocs(); @@ -836,7 +851,7 @@ void DocumentSelectParserTest::testOperators8() PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$y}[$x].value == \"structval2\"", *_doc[1], False); } -void DocumentSelectParserTest::testOperators9() +TEST_F(DocumentSelectParserTest, operators_9) { createDocs(); @@ -1047,6 +1062,11 @@ void DocumentSelectParserTest::testDocumentUpdates0() PARSEI("\"foo\" == \"foo\"", *_update[0], True); PARSEI("\"bar\" = \"a\"", *_update[0], False); PARSEI("\"bar\" = \"*a*\"", *_update[0], True); + PARSEI("\"bar\" = \"**\"", *_update[0], True); + PARSEI("\"bar\" = \"***\"", *_update[0], True); + PARSEI("\"bar\" = \"****\"", *_update[0], True); + PARSEI("\"bar\" = \"???\"", *_update[0], True); + PARSEI("\"bar\" = \"????\"", *_update[0], False); PARSEI("\"bar\" = \"\"", *_update[0], False); PARSEI("\"\" = \"\"", *_update[0], True); PARSEI("\"bar\" =~ \"^a$\"", *_update[0], False); @@ -1524,4 +1544,48 @@ TEST_F(DocumentSelectParserTest, test_parse_utilities_handle_malformed_input) check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity()); } +TEST_F(DocumentSelectParserTest, imported_field_references_are_treated_as_valid_field_with_missing_value) { + const DocumentType* type = _repo->getDocumentType("with_imported"); + ASSERT_TRUE(type != nullptr); + Document doc(*type, DocumentId("id::with_imported::foo")); + + PARSE("with_imported.my_imported_field == null", doc, True); + PARSE("with_imported.my_imported_field != null", doc, False); + PARSE("with_imported.my_imported_field", doc, False); + // Only (in)equality operators are well defined for null values; everything else becomes Invalid. + PARSE("with_imported.my_imported_field > 0", doc, Invalid); +} + +TEST_F(DocumentSelectParserTest, imported_field_references_only_support_for_simple_expressions) { + const DocumentType* type = _repo->getDocumentType("with_imported"); + ASSERT_TRUE(type != nullptr); + Document doc(*type, DocumentId("id::with_imported::foo")); + + PARSE("with_imported.my_imported_field.foo", doc, Invalid); + PARSE("with_imported.my_imported_field[0]", doc, Invalid); + PARSE("with_imported.my_imported_field{foo}", doc, Invalid); +} + +TEST_F(DocumentSelectParserTest, prefix_and_suffix_wildcard_globs_are_rewritten_to_optimized_form) { + using select::GlobOperator; + EXPECT_EQ(GlobOperator::convertToRegex("*foo"), "foo$"); + EXPECT_EQ(GlobOperator::convertToRegex("foo*"), "^foo"); + EXPECT_EQ(GlobOperator::convertToRegex("*foo*"), "foo"); + EXPECT_EQ(GlobOperator::convertToRegex("*"), ""); // Matches any string. + EXPECT_EQ(GlobOperator::convertToRegex("**"), ""); // Still matches any string. +} + +TEST_F(DocumentSelectParserTest, redundant_glob_wildcards_are_collapsed_into_minimal_form) { + using select::GlobOperator; + EXPECT_EQ(GlobOperator::convertToRegex("***"), ""); // Even still matches any string. + EXPECT_EQ(GlobOperator::convertToRegex("**foo**"), "foo"); + EXPECT_EQ(GlobOperator::convertToRegex("foo***"), "^foo"); + EXPECT_EQ(GlobOperator::convertToRegex("***foo"), "foo$"); + EXPECT_EQ(GlobOperator::convertToRegex("foo**bar"), "^foo.*bar$"); + EXPECT_EQ(GlobOperator::convertToRegex("**foo*bar**"), "foo.*bar"); + EXPECT_EQ(GlobOperator::convertToRegex("**foo***bar**"), "foo.*bar"); + EXPECT_EQ(GlobOperator::convertToRegex("*?*"), "."); + EXPECT_EQ(GlobOperator::convertToRegex("*?*?*?*"), "..*..*."); // Don't try this at home, kids! +} + } // document diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index d75a16ea1a6..968470e9693 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/document/datatype/annotationreferencedatatype.h> #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> @@ -9,6 +8,9 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> @@ -28,12 +30,16 @@ using namespace fieldvalue; TEST(DocumentTest, testSizeOf) { + EXPECT_EQ(24u, sizeof(std::vector<char>)); + EXPECT_EQ(24u, sizeof(vespalib::alloc::Alloc)); + EXPECT_EQ(24u, sizeof(ByteBuffer)); + EXPECT_EQ(32u, sizeof(vespalib::GrowableByteBuffer)); EXPECT_EQ(88ul, sizeof(IdString)); EXPECT_EQ(104ul, sizeof(DocumentId)); - EXPECT_EQ(208ul, sizeof(Document)); - EXPECT_EQ(72ul, sizeof(StructFieldValue)); - EXPECT_EQ(24ul, sizeof(StructuredFieldValue)); - EXPECT_EQ(64ul, sizeof(SerializableArray)); + EXPECT_EQ(240ul, sizeof(Document)); + EXPECT_EQ(96ul, sizeof(StructFieldValue)); + EXPECT_EQ(16ul, sizeof(StructuredFieldValue)); + EXPECT_EQ(56ul, sizeof(SerializableArray)); } TEST(DocumentTest, testFieldPath) @@ -63,7 +69,7 @@ TEST(DocumentTest, testFieldPath) class Handler : public fieldvalue::IteratorHandler { public: Handler(); - ~Handler(); + ~Handler() override; const std::string & getResult() const { return _result; } private: void onPrimitive(uint32_t, const Content&) override { @@ -386,13 +392,13 @@ TEST(DocumentTest, testSimpleUsage) EXPECT_EQ(1, value.getValue(intF)->getAsInt()); EXPECT_EQ(2, value.getValue(longF)->getAsInt()); - // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + // Serialize & equality + nbostream buffer; + value.serialize(buffer); Document value2(*repo.getDocumentType("test"), DocumentId("id::test:n=3:foo")); EXPECT_TRUE(value != value2); - value2.deserialize(repo, *buffer); + value2.deserialize(repo, buffer); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); EXPECT_EQ(DocumentId("id:ns:test::1"), value2.getId()); @@ -400,15 +406,15 @@ TEST(DocumentTest, testSimpleUsage) // Various ways of removing { // By value - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); @@ -561,29 +567,30 @@ TEST(DocumentTest, testReadSerializedFile) int fd = open(TEST_PATH("data/serializejava.dat").c_str(), O_RDONLY); size_t len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != (ssize_t)len) { + if (read(fd, buf.get(), len) != (ssize_t)len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); verifyJavaDocument(doc2); - EXPECT_EQ(len, buf2->getPos()); - EXPECT_TRUE(memcmp(buf2->getBuffer(), buf.getBuffer(), buf2->getPos()) == 0); + EXPECT_TRUE(buf2.empty()); + buf2.rp(0); + EXPECT_EQ(len, buf2.size()); doc2.setValue("stringfield", StringFieldValue("hei")); - std::unique_ptr<ByteBuffer> buf3 = doc2.serialize(); - EXPECT_TRUE(len != buf3->getPos()); + nbostream buf3 = doc2.serialize(); + EXPECT_TRUE(len != buf3.size()); } TEST(DocumentTest, testReadSerializedFileCompressed) @@ -595,14 +602,15 @@ TEST(DocumentTest, testReadSerializedFileCompressed) int fd = open(TEST_PATH("data/serializejava-compressed.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); } @@ -706,26 +714,25 @@ TEST(DocumentTest,testReadSerializedAllVersions) // you can copy this current to new test for new version) { //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_EQ(buf->getLength(), buf->getPos()); + nbostream buf = doc.serialize(); int fd = open(TEST_PATH("data/document-cpp-currentversion-uncompressed.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); } { CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_TRUE(buf->getPos() <= buf->getLength()); + nbostream buf = doc.serialize(); + EXPECT_TRUE(buf.size() <= buf.capacity()); int fd = open(TEST_PATH("data/document-cpp-currentversion-lz4-9.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg); } @@ -745,14 +752,15 @@ TEST(DocumentTest,testReadSerializedAllVersions) } int fd = open(tests[i]._dataFile.c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { - throw vespalib::Exception("read failed"); - } + if (read(fd, buf.get(), len) != len) { + throw vespalib::Exception("read failed"); + } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); IntFieldValue intVal; EXPECT_TRUE(doc.getValue(doc.getField("intfield"), intVal)); @@ -805,27 +813,14 @@ TEST(DocumentTest,testReadSerializedAllVersions) EXPECT_EQ(199, wset.get(StringFieldValue("Weighted 1"))); // Check that serialization doesn't cause any problems. - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); } } size_t getSerializedSize(const Document &doc) { - return doc.serialize()->getLength(); -} - -size_t getSerializedSizeHeader(const Document &doc) { - nbostream stream; - doc.serializeHeader(stream); - return stream.size(); -} - -size_t getSerializedSizeBody(const Document &doc) { - nbostream stream; - doc.serializeBody(stream); - return stream.size(); + return doc.serialize().size(); } TEST(DocumentTest, testGenerateSerializedFile) @@ -864,30 +859,21 @@ TEST(DocumentTest, testGenerateSerializedFile) map.put(StringFieldValue("foo2"), StringFieldValue("bar2")); doc.setValue("mapfield", map); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); + nbostream buf = doc.serialize(); const std::string serializedDir = TEST_PATH("../test/document/"); int fd = open((serializedDir + "/serializecpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.peek(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); - ByteBuffer hBuf(getSerializedSizeHeader(doc)); + vespalib::nbostream hBuf; doc.serializeHeader(hBuf); fd = open((serializedDir + "/serializecppsplit_header.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, hBuf.getBuffer(), hBuf.getPos()) != (ssize_t)hBuf.getPos()) { - throw vespalib::Exception("write failed"); - } - close(fd); - - ByteBuffer bBuf(getSerializedSizeBody(doc)); - doc.serializeBody(bBuf); - fd = open((serializedDir+ "/serializecppsplit_body.dat").c_str(), - O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, bBuf.getBuffer(), bBuf.getPos()) != (ssize_t)bBuf.getPos()) { + if (write(fd, hBuf.peek(), hBuf.size()) != (ssize_t)hBuf.size()) { throw vespalib::Exception("write failed"); } close(fd); @@ -895,62 +881,30 @@ TEST(DocumentTest, testGenerateSerializedFile) CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - ByteBuffer lz4buf(getSerializedSize(doc)); - - doc.serialize(lz4buf); - lz4buf.flip(); + nbostream lz4buf = doc.serialize(); fd = open((serializedDir + "/serializecpp-lz4-level9.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, lz4buf.getBufferAtPos(), lz4buf.getRemaining()) != (ssize_t)lz4buf.getRemaining()) { + if (write(fd, lz4buf.data(), lz4buf.size()) != (ssize_t)lz4buf.size()) { throw vespalib::Exception("write failed"); } close(fd); } -TEST(DocumentTest, testGetURIFromSerialized) -{ - TestDocRepo test_repo; - Document doc(*test_repo.getDocumentType("testdoctype1"), DocumentId("id:ns:testdoctype1::1")); - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - EXPECT_EQ( - vespalib::string("id:ns:testdoctype1::1"), - Document::getIdFromSerialized(*serialized).toString()); - - EXPECT_EQ(vespalib::string("testdoctype1"), - Document::getDocTypeFromSerialized( - test_repo.getTypeRepo(), - *serialized)->getName()); - } - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - Document doc2(test_repo.getTypeRepo(), *serialized, false, NULL); - EXPECT_EQ(vespalib::string("id:ns:testdoctype1::1"), doc2.getId().toString()); - EXPECT_EQ(vespalib::string("testdoctype1"), doc2.getType().getName()); - } -} - TEST(DocumentTest, testBogusserialize) { TestDocRepo test_repo; try { - auto buf = std::make_unique<ByteBuffer>("aoifjweprjwoejr203r+2+4r823++!",100); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("aoifjweprjwoejr203r+2+4r823++!",100); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing bogus data"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Unrecognized serialization version")); } try { - auto buf = std::make_unique<ByteBuffer>("",0); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("",0); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing empty buffer"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Buffer out of bounds")); @@ -967,24 +921,23 @@ TEST(DocumentTest, testCRC32) uint32_t crc = doc.calculateChecksum(); EXPECT_EQ(3987392271u, crc); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); + nbostream buf = doc.serialize(); int pos = 30; // Corrupt serialization. - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; // Create document. Byte corrupted above is in data area and // shouldn't fail deserialization. try { - Document doc2(test_repo.getTypeRepo(), *buf); - buf->setPos(0); + Document doc2(test_repo.getTypeRepo(), buf); + buf.rp(0); EXPECT_TRUE(crc != doc2.calculateChecksum()); } catch (document::DeserializeException& e) { EXPECT_TRUE(false); } // Return original value and retry - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; /// \todo TODO (was warning): Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added. } @@ -1001,13 +954,12 @@ TEST(DocumentTest, testHasChanged) // Still changed after setting a value of course. EXPECT_TRUE(doc.hasChanged()); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); - + nbostream buf; + doc.serialize(buf); // Setting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); EXPECT_TRUE(!doc2.hasChanged()); doc2.set("headerval", 13); @@ -1015,16 +967,16 @@ TEST(DocumentTest, testHasChanged) } // Overwriting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.set("hstringval", "bla bla bla bla bla"); EXPECT_TRUE(doc2.hasChanged()); } // Clearing value tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.clear(); EXPECT_TRUE(doc2.hasChanged()); @@ -1032,35 +984,6 @@ TEST(DocumentTest, testHasChanged) // Add more tests here when we allow non-const refs to internals } -TEST(DocumentTest, testSplitSerialization) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - doc->set("headerval", 50); - - ByteBuffer buf(getSerializedSizeHeader(*doc)); - doc->serializeHeader(buf); - buf.flip(); - - ByteBuffer buf2(getSerializedSizeBody(*doc)); - doc->serializeBody(buf2); - buf2.flip(); - - EXPECT_EQ(size_t(65), buf.getLength()); - EXPECT_EQ(size_t(73), buf2.getLength()); - - Document headerDoc(testDocMan.getTypeRepo(), buf); - EXPECT_TRUE(headerDoc.hasValue("headerval")); - EXPECT_TRUE(!headerDoc.hasValue("content")); - - buf.setPos(0); - Document fullDoc(testDocMan.getTypeRepo(), buf, buf2); - EXPECT_TRUE(fullDoc.hasValue("headerval")); - EXPECT_TRUE(fullDoc.hasValue("content")); - - EXPECT_EQ(*doc, fullDoc); -} - TEST(DocumentTest, testSliceSerialize) { // Test that document doesn't need its own bytebuffer, such that we @@ -1076,17 +999,15 @@ TEST(DocumentTest, testSliceSerialize) val.add(RawFieldValue("hei der", 7)); doc2->setValue(doc2->getField("rawarray"), val); - ByteBuffer buf(getSerializedSize(*doc) + getSerializedSize(*doc2)); - doc->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + nbostream buf = doc->serialize(); + EXPECT_EQ(getSerializedSize(*doc), buf.size()); doc2->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); - buf.flip(); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.size()); Document doc3(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc), buf.rp()); Document doc4(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.rp()); EXPECT_EQ(*doc, doc3); EXPECT_EQ(*doc2, doc4); @@ -1102,21 +1023,19 @@ TEST(DocumentTest, testCompression) doc->setValue("hstringval", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc->serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed = doc->serialize(); CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf_lz4 = doc->serialize(); - buf_lz4->flip(); + nbostream buf_lz4 = doc->serialize(); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg); - EXPECT_TRUE(buf_lz4->getRemaining() < buf_uncompressed->getRemaining()); + EXPECT_TRUE(buf_lz4.size() < buf_uncompressed.size()); - Document doc_lz4(testDocMan.getTypeRepo(), *buf_lz4); + Document doc_lz4(testDocMan.getTypeRepo(), buf_lz4); EXPECT_EQ(*doc, doc_lz4); } @@ -1136,10 +1055,10 @@ TEST(DocumentTest, testCompressionConfigured) for (int i = 0; i < 8; ++i) { bigString += bigString; } doc_uncompressed.setValue("stringfield", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc_uncompressed.serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed; + doc_uncompressed.serialize(buf_uncompressed); - size_t uncompressedSize = buf_uncompressed->getRemaining(); + size_t uncompressedSize = buf_uncompressed.size(); DocumenttypesConfigBuilderHelper builder2; builder2.document(43, "serializetest", @@ -1151,22 +1070,22 @@ TEST(DocumentTest, testCompressionConfigured) 9, 99, 0)); DocumentTypeRepo repo2(builder2.config()); - Document doc(repo2, *buf_uncompressed); + Document doc(repo2, buf_uncompressed); - std::unique_ptr<ByteBuffer> buf_compressed = doc.serialize(); - buf_compressed->flip(); - size_t compressedSize = buf_compressed->getRemaining(); + nbostream buf_compressed; + doc.serialize(buf_compressed); + size_t compressedSize = buf_compressed.size(); EXPECT_TRUE(compressedSize < uncompressedSize); - Document doc2(repo2, *buf_compressed); + Document doc2(repo2, buf_compressed); - std::unique_ptr<ByteBuffer> buf_compressed2 = doc2.serialize(); - buf_compressed2->flip(); + nbostream buf_compressed2; + doc2.serialize(buf_compressed2); - EXPECT_EQ(compressedSize, buf_compressed2->getRemaining()); + EXPECT_EQ(compressedSize, buf_compressed2.size()); - Document doc3(repo2, *buf_compressed2); + Document doc3(repo2, buf_compressed2); EXPECT_EQ(doc2, doc_uncompressed); EXPECT_EQ(doc2, doc3); @@ -1198,59 +1117,31 @@ TEST(DocumentTest, testUnknownEntries) doc1.setValue(field3, IntFieldValue(3)); doc1.setValue(field4, IntFieldValue(4)); - uint32_t headerLen = getSerializedSizeHeader(doc1); - document::ByteBuffer header(headerLen); - doc1.serializeHeader(header); - header.flip(); - - uint32_t bodyLen = getSerializedSizeBody(doc1); - document::ByteBuffer body(bodyLen); - doc1.serializeBody(body); - body.flip(); - - uint32_t totalLen = getSerializedSize(doc1); - document::ByteBuffer total(totalLen); - doc1.serialize(total); - total.flip(); + vespalib::nbostream os; + doc1.serialize(os); Document doc2; - doc2.deserialize(repo, total); - - Document doc3; - doc3.deserializeHeader(repo, header); - doc3.deserializeBody(repo, body); + doc2.deserialize(repo, os); EXPECT_EQ(std::string( "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" "<int3>3</int3>\n" "<int4>4</int4>\n" "</document>"), doc2.toXml()); - EXPECT_EQ(std::string( - "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" - "<int3>3</int3>\n" - "<int4>4</int4>\n" - "</document>"), doc3.toXml()); EXPECT_EQ(3, doc2.getValue(field3)->getAsInt()); EXPECT_EQ(4, doc2.getValue(field4)->getAsInt()); - EXPECT_EQ(3, doc3.getValue(field3)->getAsInt()); - EXPECT_EQ(4, doc3.getValue(field4)->getAsInt()); // The fields are actually accessible as long as you ask with field of // correct type. EXPECT_TRUE(doc2.hasValue(field1)); EXPECT_TRUE(doc2.hasValue(field2)); - EXPECT_TRUE(doc3.hasValue(field1)); - EXPECT_TRUE(doc3.hasValue(field2)); EXPECT_EQ(1, doc2.getValue(field1)->getAsInt()); EXPECT_EQ(2, doc2.getValue(field2)->getAsInt()); - EXPECT_EQ(1, doc3.getValue(field1)->getAsInt()); - EXPECT_EQ(2, doc3.getValue(field2)->getAsInt()); EXPECT_EQ(size_t(2), doc2.getSetFieldCount()); - EXPECT_EQ(size_t(2), doc3.getSetFieldCount()); } TEST(DocumentTest, testAnnotationDeserialization) @@ -1280,14 +1171,15 @@ TEST(DocumentTest, testAnnotationDeserialization) int fd = open(TEST_PATH("data/serializejavawithannotations.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream1(buf.get(), len); + Document doc(repo, stream1); StringFieldValue strVal; EXPECT_TRUE(doc.getValue(doc.getField("story"), strVal)); @@ -1326,14 +1218,6 @@ TEST(DocumentTest, testAnnotationDeserialization) EXPECT_EQ((int64_t)2384LL, longVal.getAsLong()); } -TEST(DocumentTest, testGetSerializedSize) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - - EXPECT_EQ(getSerializedSize(*doc), doc->getSerializedSize()); -} - TEST(DocumentTest, testDeserializeMultiple) { TestDocRepo testDocRepo; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 47a529adfc8..18001c35da5 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -43,14 +43,13 @@ namespace document { namespace { -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } nbostream serialize(const ValueUpdate & update) @@ -83,25 +82,25 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } void -writeBufferToFile(const ByteBuffer &buf, const vespalib::string &fileName) +writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.getBuffer(), buf.getPos()); + file.write(buf.data(), buf.size()); assert(file.good()); file.close(); } -ByteBuffer::UP +nbostream readBufferFromFile(const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); auto size = file.tellg(); - auto result = std::make_unique<ByteBuffer>(size); file.seekg(0); - file.read(result->getBuffer(), size); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); assert(file.good()); file.close(); - return result; + return nbostream(std::move(buf), size); } } @@ -132,9 +131,8 @@ TEST(DocumentUpdateTest, testSimpleUsage) // Test that a document update can be serialized DocumentUpdate docUpdate(repo, *docType, DocumentId("id:ns:test::1")); docUpdate.addUpdate(fieldUpdateCopy); - ByteBuffer::UP docBuf = serializeHEAD(docUpdate); - docBuf->flip(); - auto docUpdateCopy(DocumentUpdate::createHEAD(repo, nbostream(docBuf->getBufferAtPos(), docBuf->getRemaining()))); + nbostream docBuf = serializeHEAD(docUpdate); + auto docUpdateCopy(DocumentUpdate::createHEAD(repo, docBuf)); // Create a test document Document doc(*docType, DocumentId("id:ns:test::1")); @@ -236,7 +234,7 @@ TEST(DocumentUpdateTest, testUpdateArray) // Create a document. TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); // Assign array field. ArrayFieldValue myarray(doc->getType().getField("tags").getDataType()); @@ -459,8 +457,7 @@ TEST(DocumentUpdateTest, testReadSerializedFile) const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto buf = readBufferFromFile("data/serializeupdatejava.dat"); - nbostream is(buf->getBufferAtPos(), buf->getRemaining()); + auto is = readBufferFromFile("data/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -539,8 +536,8 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2))) .addUpdate(MapValueUpdate(StringFieldValue("foo"), ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2)))); - ByteBuffer::UP buf(serializeHEAD(upd)); - writeBufferToFile(*buf, "data/serializeupdatecpp.dat"); + nbostream buf(serializeHEAD(upd)); + writeBufferToFile(buf, "data/serializeupdatecpp.dat"); } @@ -549,7 +546,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("headerval")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); // Assign a float value to an int field. DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); @@ -561,7 +558,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) update.applyTo(*doc); // Verify that the field is NOT set in the document. - EXPECT_EQ((document::FieldValue*)NULL, + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); } @@ -569,7 +566,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate())); @@ -875,6 +872,13 @@ struct TensorUpdateFixture { EXPECT_EQ(actTensor, expTensor); } + void assertTensorNull() { + auto field = getTensor(); + auto tensor_field = dynamic_cast<TensorFieldValue*>(field.get()); + ASSERT_TRUE(tensor_field); + EXPECT_TRUE(tensor_field->getAsTensorPtr().get() == nullptr); + } + void assertTensor(const TensorSpec &expSpec) { auto expTensor = makeTensor(expSpec); assertTensor(*expTensor); @@ -889,6 +893,19 @@ struct TensorUpdateFixture { assertTensor(expTensor); } + void assertApplyUpdateNonExisting(const ValueUpdate &update, + const TensorSpec &expTensor) { + applyUpdate(update); + assertDocumentUpdated(); + assertTensor(expTensor); + } + + void assertApplyUpdateNonExisting(const ValueUpdate &update) { + applyUpdate(update); + assertDocumentUpdated(); + assertTensorNull(); + } + template <typename ValueUpdateType> void assertRoundtripSerialize(const ValueUpdateType &valueUpdate) { testRoundtripSerialize(valueUpdate, tensorDataType); @@ -936,6 +953,16 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) .add({{"x", "c"}}, 7)); } +TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorAddUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7))), + + f.spec().add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7)); +} + TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) { TensorUpdateFixture f; @@ -947,6 +974,12 @@ TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) f.spec().add({{"x", "a"}}, 2)); } +TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorRemoveUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 1)))); +} + TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) { TensorUpdateFixture f; @@ -973,6 +1006,13 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) .add({{"x", "b"}}, 15)); } +TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorModifyUpdate(TensorModifyUpdate::Operation::ADD, + f.makeTensor(f.spec().add({{"x", "b"}}, 5)))); +} + TEST(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; @@ -1110,13 +1150,12 @@ struct TensorUpdateSerializeFixture { } void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { - ByteBuffer::UP buf = serializeHEAD(update); - writeBufferToFile(*buf, fileName); + nbostream buf = serializeHEAD(update); + writeBufferToFile(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto buf = readBufferFromFile(fileName); - nbostream stream(buf->getBufferAtPos(), buf->getRemaining()); + auto stream = readBufferFromFile(fileName); return DocumentUpdate::createHEAD(*repo, stream); } @@ -1196,10 +1235,9 @@ TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeseriali { CreateIfNonExistentFixture f; - ByteBuffer::UP buf(serializeHEAD(*f.update)); - buf->flip(); + nbostream buf(serializeHEAD(*f.update)); - DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), *buf); + DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), buf); EXPECT_EQ(*f.update, *deserialized); EXPECT_TRUE(deserialized->getCreateIfNonExistent()); } @@ -1231,9 +1269,8 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) ArrayUpdateFixture f; auto buffer = serializeHEAD(*f.update); - buffer->flip(); - auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), *buffer); + auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), buffer); EXPECT_EQ(*f.update, *deserialized); } diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 82443f13716..213ac2e5432 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -19,6 +19,7 @@ #include <gtest/gtest.h> using vespalib::Identifiable; +using vespalib::nbostream; using namespace document::config_builder; namespace document { @@ -133,23 +134,21 @@ createTestDocument(const DocumentTypeRepo &repo) return doc; } -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { vespalib::nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - ByteBuffer::UP bb(serializeHEAD(a)); - bb->flip(); - DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); + auto bb(serializeHEAD(a)); + DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, bb)); - EXPECT_EQ(size_t(0), bb->getRemaining()); + EXPECT_EQ(size_t(0), bb.size()); EXPECT_EQ(a.getId().toString(), b->getId().toString()); EXPECT_EQ(a.getUpdates().size(), b->getUpdates().size()); for (size_t i(0); i < a.getUpdates().size(); i++) { @@ -157,8 +156,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { const FieldUpdate & ub = b->getUpdates()[i]; EXPECT_EQ(&ua.getField(), &ub.getField()); - EXPECT_EQ(ua.getUpdates().size(), - ub.getUpdates().size()); + EXPECT_EQ(ua.getUpdates().size(), ub.getUpdates().size()); for (size_t j(0); j < ua.getUpdates().size(); j++) { EXPECT_EQ(ua.getUpdates()[j]->getType(), ub.getUpdates()[j]->getType()); } @@ -1073,14 +1071,14 @@ TEST_F(FieldPathUpdateTestCase, testReadSerializedFile) int fd = open(TEST_PATH("data/serialize-fieldpathupdate-java.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, buf)); + DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, nbostream(std::move(buf), len))); DocumentUpdate& upd(*updp); DocumentUpdate::UP compare(createDocumentUpdateForSerialization(repo)); @@ -1094,11 +1092,11 @@ TEST_F(FieldPathUpdateTestCase, testGenerateSerializedFile) // Tests nothing, only generates a file for java test DocumentUpdate::UP upd(createDocumentUpdateForSerialization(repo)); - ByteBuffer::UP buf(serializeHEAD(*upd)); + nbostream buf(serializeHEAD(*upd)); int fd = open(TEST_PATH("data/serialize-fieldpathupdate-cpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.data(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 58592e50cfe..8a5daf05f05 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -14,9 +14,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -89,20 +88,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Serialization Type t; - std::unique_ptr<ByteBuffer> buf(smallest.serialize()); - buf->flip(); - deserialize(*buf, t); + nbostream buf(smallest.serialize()); + deserialize(buf, t); EXPECT_EQ(smallest, t); buf = medium1.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(medium1, t); EXPECT_EQ(medium2, t); buf = largest.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(largest, t); // Assignment @@ -160,20 +156,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Test that a just deserialized value can be serialized again // (literals have lazy deserialization so behaves diff then value = "foo"; - std::unique_ptr<ByteBuffer> buf(value.serialize()); - buf->flip(); + nbostream buf(value.serialize()); Literal value2("Other"); - deserialize(*buf, value2); + deserialize(buf, value2); buf = value2.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(value, value2); // Verify that get value ref gives us ref within original bytebuffer // (operator== use above should not modify this) buf = value.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(size_t(3), value2.getValueRef().size()); // Zero termination diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index b263ad75930..0bc80ebcd16 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -532,6 +532,47 @@ TEST("Reference fields are resolved to correct reference type") { EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType()); } +TEST("Config with no imported fields has empty imported fields set in DocumentType") { + DocumenttypesConfigBuilderHelper builder; + builder.document(doc_type_id, type_name, + Struct(header_name), Struct(body_name)); + DocumentTypeRepo repo(builder.config()); + const auto *type = repo.getDocumentType(doc_type_id); + ASSERT_TRUE(type != nullptr); + EXPECT_TRUE(type->imported_field_names().empty()); + EXPECT_FALSE(type->has_imported_field_name("foo")); +} + +TEST("Configured imported field names are available in the DocumentType") { + const int type_2_id = doc_type_id + 1; + // Note: we cheat a bit by specifying imported field names in types that have no + // reference fields. Add to test if we add config read-time validation of this. :) + DocumenttypesConfigBuilderHelper builder; + // Type with one imported field + builder.document(doc_type_id, type_name, + Struct(header_name), Struct(body_name)) + .imported_field("my_cool_field"); + // Type with two imported fields + builder.document(type_2_id, type_name_2, + Struct(header_name_2), Struct(body_name_2)) + .imported_field("my_awesome_field") + .imported_field("my_swag_field"); + + DocumentTypeRepo repo(builder.config()); + const auto* type = repo.getDocumentType(doc_type_id); + ASSERT_TRUE(type != nullptr); + EXPECT_EQUAL(1u, type->imported_field_names().size()); + EXPECT_TRUE(type->has_imported_field_name("my_cool_field")); + EXPECT_FALSE(type->has_imported_field_name("my_awesome_field")); + + type = repo.getDocumentType(type_2_id); + ASSERT_TRUE(type != nullptr); + EXPECT_EQUAL(2u, type->imported_field_names().size()); + EXPECT_TRUE(type->has_imported_field_name("my_awesome_field")); + EXPECT_TRUE(type->has_imported_field_name("my_swag_field")); + EXPECT_FALSE(type->has_imported_field_name("my_cool_field")); +} + namespace { const TensorDataType & diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 7feea4e51d2..2ad06dc93de 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -535,12 +535,12 @@ TEST("requireThatDocumentCanBeSerialized") { uint32_t size; stream >> read_version >> size; EXPECT_EQUAL(serialization_version, read_version); - EXPECT_EQUAL(70u, size); + EXPECT_EQUAL(64u, size); EXPECT_EQUAL(doc_id.getScheme().toString(), stream.peek()); stream.adjustReadPos(doc_id.getScheme().toString().size() + 1); uint8_t content_code; stream >> content_code; - EXPECT_EQUAL(0x07, content_code); + EXPECT_EQUAL(0x03, content_code); EXPECT_EQUAL(type.getName(), stream.peek()); stream.adjustReadPos(type.getName().size() + 1); stream >> read_version; @@ -912,7 +912,7 @@ void DeserializedTensorDoc::setup(const DocumentTypeRepo &docTypeRepo, const vespalib::nbostream &blob) { vespalib::nbostream wrapStream(blob.peek(), blob.size()); - _doc = std::make_unique<Document>(docTypeRepo, wrapStream, nullptr); + _doc = std::make_unique<Document>(docTypeRepo, wrapStream); _fieldValue = _doc->getValue(tensor_field_name); } diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp index 76cf065a36a..9cf4b38be91 100644 --- a/document/src/tests/structfieldvaluetest.cpp +++ b/document/src/tests/structfieldvaluetest.cpp @@ -28,10 +28,9 @@ protected: namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value, const FixedTypeRepo &repo) +void deserialize(nbostream & stream, T &value, const FixedTypeRepo &repo) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); } @@ -61,13 +60,11 @@ TEST_F(StructFieldValueTest, testEmptyStruct) StructFieldValue value(type); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value == value2); } @@ -101,14 +98,12 @@ TEST_F(StructFieldValueTest, testStruct) EXPECT_EQ(2, value.getValue(longF)->getAsInt()); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); @@ -116,15 +111,15 @@ TEST_F(StructFieldValueTest, testStruct) // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); diff --git a/document/src/tests/testbytebuffer.cpp b/document/src/tests/testbytebuffer.cpp index 17807fb4ff5..7d412be5ba3 100644 --- a/document/src/tests/testbytebuffer.cpp +++ b/document/src/tests/testbytebuffer.cpp @@ -3,12 +3,14 @@ #include <vespa/document/util/stringutil.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldvalue/serializablearray.h> -#include <iostream> -#include <vespa/vespalib/util/macro.h> #include <vespa/document/util/bufferexceptions.h> +#include <vespa/vespalib/util/macro.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <gtest/gtest.h> + using namespace document; +using vespalib::GrowableByteBuffer; namespace { @@ -22,111 +24,28 @@ void assign(S &lhs, const S &rhs) TEST(ByteBuffer_Test, test_constructors) { - ByteBuffer* simple=new ByteBuffer(); - delete simple; - - ByteBuffer* less_simple=new ByteBuffer("hei",3); - EXPECT_TRUE(strcmp(less_simple->getBufferAtPos(),"hei")==0); - delete less_simple; -} - -TEST(ByteBuffer_Test, test_assignment_operator) -{ - try { - ByteBuffer b1; - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); - - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - int test = 0; - b2.flip(); - b2.getInt(test); - EXPECT_EQ(1,test); - b2.getInt(test); - EXPECT_EQ(2,test); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - // Test Selfassignment == no change - // - assign(b2, b2); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - ByteBuffer b3; - // Empty - b2 = b3; - - EXPECT_EQ((size_t) 0,b2.getPos()); - EXPECT_EQ((size_t) 0,b2.getLength()); - EXPECT_EQ((size_t) 0,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } + ByteBuffer less_simple("hei",3); + EXPECT_TRUE(strcmp(less_simple.getBufferAtPos(),"hei")==0); } TEST(ByteBuffer_Test, test_copy_constructor) { try { - // Empty buffer first - ByteBuffer b1; - ByteBuffer b2(b1); - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); + GrowableByteBuffer gb(100); + gb.putInt(1); + gb.putInt(2); + ByteBuffer b1(gb.getBuffer(), gb.position()); ByteBuffer b2(b1); EXPECT_EQ(b1.getPos(),b2.getPos()); EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); int test = 0; - b2.flip(); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(1,test); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(2,test); } catch (std::exception &e) { @@ -134,327 +53,6 @@ TEST(ByteBuffer_Test, test_copy_constructor) } } -TEST(ByteBuffer_Test, test_slice) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer* slice = new ByteBuffer; - slice->sliceFrom(*newBuf, 0,newBuf->getLength()); - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice->getBufferAtPos(),"hei der")==0); - - ByteBuffer* slice2 = new ByteBuffer; - slice2->sliceFrom(*slice, 4, slice->getLength()); - delete slice; - slice = NULL; - - EXPECT_TRUE(strcmp(slice2->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2->getBuffer(),"hei der")==0); - delete slice2; - slice2 = NULL; - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - ByteBuffer* slice3=new ByteBuffer; - ByteBuffer* slice4=new ByteBuffer; - - slice3->sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice4->sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice3->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice3; - slice3 = NULL; - - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice4; - slice4 = NULL; -} - -TEST(ByteBuffer_Test, test_slice2) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer slice; - slice.sliceFrom(*newBuf, 0, newBuf->getLength()); - - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"hei der")==0); - - ByteBuffer slice2; - slice2.sliceFrom(slice, 4, slice.getLength()); - - EXPECT_TRUE(strcmp(slice2.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - - slice.sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice2.sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); -} - - -TEST(ByteBuffer_Test, test_putGetFlip) -{ - ByteBuffer* newBuf=new ByteBuffer(100); - - try { - newBuf->putInt(10); - int test; - newBuf->flip(); - - newBuf->getInt(test); - EXPECT_TRUE(test==10); - - newBuf->clear(); - newBuf->putDouble(3.35); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==sizeof(double)); - double test2; - newBuf->getDouble(test2); - EXPECT_TRUE(test2==3.35); - - newBuf->clear(); - newBuf->putBytes("heisann",8); - newBuf->putInt(4); - EXPECT_TRUE(newBuf->getPos()==12); - EXPECT_TRUE(newBuf->getLength()==100); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==12); - - char testStr[12]; - newBuf->getBytes(testStr, 8); - EXPECT_TRUE(strcmp(testStr,"heisann")==0); - newBuf->getInt(test); - EXPECT_TRUE(test==4); - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - delete newBuf; -} - - -TEST(ByteBuffer_Test, test_NumberEncodings) -{ - ByteBuffer* buf=new ByteBuffer(1024); - - // Check 0 - buf->putInt1_2_4Bytes(124); - buf->putInt2_4_8Bytes(124); - buf->putInt1_4Bytes(124); - // Check 1 - buf->putInt1_2_4Bytes(127); - buf->putInt2_4_8Bytes(127); - buf->putInt1_4Bytes(127); - // Check 2 - buf->putInt1_2_4Bytes(128); - buf->putInt2_4_8Bytes(128); - buf->putInt1_4Bytes(128); - // Check 3 - buf->putInt1_2_4Bytes(255); - buf->putInt2_4_8Bytes(255); - buf->putInt1_4Bytes(255); - // Check 4 - buf->putInt1_2_4Bytes(256); - buf->putInt2_4_8Bytes(256); - buf->putInt1_4Bytes(256); - // Check 5 - buf->putInt1_2_4Bytes(0); - buf->putInt2_4_8Bytes(0); - buf->putInt1_4Bytes(0); - // Check 6 - buf->putInt1_2_4Bytes(1); - buf->putInt2_4_8Bytes(1); - buf->putInt1_4Bytes(1); - - // Check 7 - try { - buf->putInt1_2_4Bytes(0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - buf->putInt2_4_8Bytes(0x7FFFFFFFll); - buf->putInt1_4Bytes(0x7FFFFFFF); - - try { - buf->putInt2_4_8Bytes(0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - buf->putInt1_2_4Bytes(0x7FFF); - // Check 8 - buf->putInt2_4_8Bytes(0x7FFFll); - buf->putInt1_4Bytes(0x7FFF); - buf->putInt1_2_4Bytes(0x7F); - // Check 9 - buf->putInt2_4_8Bytes(0x7Fll); - buf->putInt1_4Bytes(0x7F); - - try { - buf->putInt1_2_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt1_2_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - uint32_t endWritePos = buf->getPos(); - buf->setPos(0); - - int32_t tmp32; - int64_t tmp64; - - // Check 0 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)124, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - // Check 1 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)127, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - // Check 2 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)128, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - // Check 3 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)255, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - // Check 4 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)256, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - // Check 5 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - // Check 6 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)1, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - // Check 7 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFFFFFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFFFFFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - // Check 8 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - // Check 9 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7F, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - - uint32_t endReadPos = buf->getPos(); - EXPECT_EQ(endWritePos, endReadPos); - - delete buf; -} - -TEST(ByteBuffer_Test, test_NumberLengths) -{ - ByteBuffer b; - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(127)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(128)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(129)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(255)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(256)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(0x7FFFFFFF)); - - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(0)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(1)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(4)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(31)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(126)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(32767)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32768)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32769)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(1030493)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(0x3FFFFFFF)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000000)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000001)); - - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(16383)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16384)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16385)); -} - TEST(ByteBuffer_Test, test_SerializableArray) { SerializableArray array; @@ -464,6 +62,6 @@ TEST(ByteBuffer_Test, test_SerializableArray) EXPECT_EQ(4ul, array.get(0).size()); EXPECT_EQ(copy.get(0).size(), array.get(0).size()); EXPECT_TRUE(copy.get(0).c_str() != array.get(0).c_str()); - EXPECT_EQ(0, strcmp(copy.get(0).c_str(), array.get(0).c_str())); + EXPECT_EQ(0, strncmp(copy.get(0).c_str(), array.get(0).c_str(), 4)); EXPECT_EQ(16ul, sizeof(SerializableArray::Entry)); } diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 1ec52791a4a..71dc9f1d1a9 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -16,9 +16,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -94,26 +93,25 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) EXPECT_EQ(6, value.get(IntFieldValue(3))); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); WeightedSetFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(buffer, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); EXPECT_EQ(size_t(3), value2.size()); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/vespa/document/base/documentid.cpp b/document/src/vespa/document/base/documentid.cpp index 58d9a83fc5b..182a96985cd 100644 --- a/document/src/vespa/document/base/documentid.cpp +++ b/document/src/vespa/document/base/documentid.cpp @@ -29,7 +29,7 @@ DocumentId::DocumentId(vespalib::nbostream & is) DocumentId::DocumentId(const DocumentId & rhs) = default; DocumentId & DocumentId::operator = (const DocumentId & rhs) = default; -DocumentId::~DocumentId() = default; +DocumentId::~DocumentId() noexcept = default; vespalib::string DocumentId::toString() const { diff --git a/document/src/vespa/document/base/documentid.h b/document/src/vespa/document/base/documentid.h index 2e4f54b43d3..5dcda838623 100644 --- a/document/src/vespa/document/base/documentid.h +++ b/document/src/vespa/document/base/documentid.h @@ -35,11 +35,11 @@ public: DocumentId(); DocumentId(vespalib::nbostream & os); - DocumentId(DocumentId && rhs) = default; - DocumentId & operator = (DocumentId && rhs) = default; + DocumentId(DocumentId && rhs) noexcept = default; + DocumentId & operator = (DocumentId && rhs) noexcept = default; DocumentId(const DocumentId & rhs); DocumentId & operator = (const DocumentId & rhs); - ~DocumentId(); + ~DocumentId() noexcept ; /** * Parse the given document identifier given as string, and create an * identifier object from it. diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index 22a0058ff8c..930668b2fac 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -194,7 +194,6 @@ IdString::Offsets::compute(stringref id) for (;index < VESPA_NELEMS(_offsets); index++) { _offsets[index] = id.size() + 1; // 1 is added due to the implicitt accounting for ':' } - _offsets[MAX_COMPONENTS] = id.size() + 1; // 1 is added due to the implicitt accounting for ':' return numComponents; } diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h index 51d99251c2c..90f307553e8 100644 --- a/document/src/vespa/document/base/idstring.h +++ b/document/src/vespa/document/base/idstring.h @@ -35,7 +35,9 @@ public: vespalib::stringref getGroup() const { return vespalib::stringref(getRawId().c_str() + _groupOffset, offset(3) - _groupOffset - 1); } - vespalib::stringref getNamespaceSpecific() const { return getComponent(3); } + vespalib::stringref getNamespaceSpecific() const { + return vespalib::stringref(_rawId.c_str() + offset(3), _rawId.size() - offset(3)); + } bool operator==(const IdString& other) const { return toString() == other.toString(); } @@ -43,8 +45,8 @@ public: const vespalib::string & toString() const { return _rawId; } private: - size_t offset(size_t index) const { return _offsets[index]; } - size_t size(size_t index) const { return std::max(0, int(_offsets[index+1]) - int(_offsets[index]) - 1); } + uint16_t offset(uint32_t index) const { return _offsets[index]; } + uint16_t size(uint32_t index) const { return std::max(0, int(offset(index+1)) - int(offset(index)) - 1); } vespalib::stringref getComponent(size_t index) const { return vespalib::stringref(_rawId.c_str() + offset(index), size(index)); } const vespalib::string & getRawId() const { return _rawId; } @@ -57,7 +59,7 @@ private: private: static constexpr uint32_t MAX_COMPONENTS = 4; Offsets(vespalib::stringref id); - uint16_t _offsets[MAX_COMPONENTS + 1]; + uint16_t _offsets[MAX_COMPONENTS]; }; vespalib::string _rawId; diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 1961f67d83c..092a29d9293 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int ## Field sets datatype[].documenttype[].fieldsets{}.fields[] string +## Imported fields (specified outside the document block in the schema) +datatype[].documenttype[].importedfield[].name string + ## Cross-document reference with ID of target document type datatype[].referencetype[].target_type_id int diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index fb3fcd8b123..0f0a9e3e37c 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -104,3 +104,6 @@ documenttype[].referencetype[].id int ## Numeric ID of the document type instances of the reference point to. documenttype[].referencetype[].target_type_id int + +## Imported fields (specified outside the document block in the schema) +documenttype[].importedfield[].name string diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp index 8198abfc7b1..c2739dc1303 100644 --- a/document/src/vespa/document/datatype/documenttype.cpp +++ b/document/src/vespa/document/datatype/documenttype.cpp @@ -18,16 +18,15 @@ namespace document { IMPLEMENT_IDENTIFIABLE(DocumentType, StructuredDataType); -DocumentType::DocumentType() -{ -} +DocumentType::DocumentType() = default; DocumentType::DocumentType(stringref name, int32_t id) : StructuredDataType(name, id), _inheritedTypes(), - _ownedFields(new StructDataType(name + ".header")), + _ownedFields(std::make_shared<StructDataType>(name + ".header")), _fields(_ownedFields.get()), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { _inheritedTypes.push_back(DataType::DOCUMENT); @@ -38,7 +37,8 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie : StructuredDataType(name, id), _inheritedTypes(), _fields(&fields), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { _inheritedTypes.push_back(DataType::DOCUMENT); @@ -48,12 +48,13 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie DocumentType::DocumentType(stringref name) : StructuredDataType(name), _inheritedTypes(), - _ownedFields(new StructDataType(name + ".header")), + _ownedFields(std::make_shared<StructDataType>(name + ".header")), _fields(_ownedFields.get()), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { - _inheritedTypes.push_back(DataType::DOCUMENT); + _inheritedTypes.emplace_back(DataType::DOCUMENT); } } @@ -61,29 +62,28 @@ DocumentType::DocumentType(stringref name, const StructDataType& fields) : StructuredDataType(name), _inheritedTypes(), _fields(&fields), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { - _inheritedTypes.push_back(DataType::DOCUMENT); + _inheritedTypes.emplace_back(DataType::DOCUMENT); } } -DocumentType::~DocumentType() -{ -} +DocumentType::~DocumentType() = default; DocumentType & -DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields) +DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields) { - _fieldSets[name] = FieldSet(name, fields); + _fieldSets[name] = FieldSet(name, std::move(fields)); return *this; } const DocumentType::FieldSet * DocumentType::getFieldSet(const vespalib::string & name) const { - FieldSetMap::const_iterator it(_fieldSets.find(name)); - return (it != _fieldSets.end()) ? & it->second : NULL; + auto it = _fieldSets.find(name); + return (it != _fieldSets.end()) ? & it->second : nullptr; } void @@ -111,37 +111,34 @@ DocumentType::inherit(const DocumentType &docType) { "Document type " + docType.toString() + " already inherits type " + toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC); } - // If we already inherits this type, there is no point in adding it - // again. + // If we already inherits this type, there is no point in adding it again. if (isA(docType)) { - // If we already directly inherits it, complain - for (std::vector<const DocumentType *>::const_iterator - it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it) - { - if (**it == docType) { + // If we already directly inherits it, complain + for (const auto* inherited : _inheritedTypes) { + if (*inherited == docType) { throw IllegalArgumentException( "DocumentType " + getName() + " already inherits " "document type " + docType.getName(), VESPA_STRLOC); } } - // Indirectly already inheriting it is oki, as this can happen - // due to inherited documents inheriting the same type. + // Indirectly already inheriting it is oki, as this can happen + // due to inherited documents inheriting the same type. LOG(info, "Document type %s inherits document type %s from multiple " "types.", getName().c_str(), docType.getName().c_str()); return; } - // Add non-conflicting types. + // Add non-conflicting types. Field::Set fs = docType._fields->getFieldSet(); - for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) { + for (const auto* field : fs) { if (!_ownedFields.get()) { _ownedFields.reset(_fields->clone()); _fields = _ownedFields.get(); } - _ownedFields->addInheritedField(**it); + _ownedFields->addInheritedField(*field); } // If we inherit default document type Document.0, remove that if adding // another parent, as that has to also inherit Document - if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) { + if ((_inheritedTypes.size() == 1) && (*_inheritedTypes[0] == *DataType::DOCUMENT)) { _inheritedTypes.clear(); } _inheritedTypes.push_back(&docType); @@ -150,10 +147,8 @@ DocumentType::inherit(const DocumentType &docType) { bool DocumentType::isA(const DataType& other) const { - for (std::vector<const DocumentType *>::const_iterator - it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it) - { - if ((*it)->isA(other)) return true; + for (const DocumentType * docType : _inheritedTypes) { + if (docType->isA(other)) return true; } return (*this == other); } @@ -161,12 +156,11 @@ DocumentType::isA(const DataType& other) const FieldValue::UP DocumentType::createFieldValue() const { - return FieldValue::UP(new Document(*this, DocumentId("doc::"))); + return std::make_unique<Document>(*this, DocumentId("id::" + getName() + "::")); } void -DocumentType::print(std::ostream& out, bool verbose, - const std::string& indent) const +DocumentType::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DocumentType(" << getName(); if (verbose) { @@ -175,8 +169,7 @@ DocumentType::print(std::ostream& out, bool verbose, out << ")"; if (verbose) { if (!_inheritedTypes.empty()) { - std::vector<const DocumentType *>::const_iterator it( - _inheritedTypes.begin()); + auto it = _inheritedTypes.begin(); out << "\n" << indent << " : "; (*it)->print(out, false, ""); while (++it != _inheritedTypes.end()) { @@ -195,17 +188,18 @@ DocumentType::operator==(const DataType& other) const { if (&other == this) return true; if (!DataType::operator==(other)) return false; - const DocumentType* o(dynamic_cast<const DocumentType*>(&other)); - if (o == 0) return false; + const auto* o(dynamic_cast<const DocumentType*>(&other)); + if (o == nullptr) return false; if (*_fields != *o->_fields) return false; if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false; - std::vector<const DocumentType *>::const_iterator it1(_inheritedTypes.begin()); - std::vector<const DocumentType *>::const_iterator it2(o->_inheritedTypes.begin()); + auto it1 = _inheritedTypes.begin(); + auto it2 = o->_inheritedTypes.begin(); while (it1 != _inheritedTypes.end()) { if (**it1 != **it2) return false; ++it1; ++it2; } + // TODO imported fields? like in the Java impl, field sets are not considered either... :I return true; } @@ -235,6 +229,14 @@ DocumentType::getFieldSet() const return _fields->getFieldSet(); } +bool DocumentType::has_imported_field_name(const vespalib::string& name) const noexcept { + return (_imported_field_names.find(name) != _imported_field_names.end()); +} + +void DocumentType::add_imported_field_name(const vespalib::string& name) { + _imported_field_names.insert(name); +} + DocumentType * DocumentType::clone() const { return new DocumentType(*this); diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h index 7fcf5dfa237..ed6e9e66ab5 100644 --- a/document/src/vespa/document/datatype/documenttype.h +++ b/document/src/vespa/document/datatype/documenttype.h @@ -12,9 +12,11 @@ #pragma once #include <vespa/document/datatype/structdatatype.h> - +#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> +#include <set> namespace document { @@ -25,12 +27,17 @@ class DocumentType : public StructuredDataType { public: class FieldSet { public: - typedef std::set<vespalib::string> Fields; - FieldSet() : _name(), _fields() {} - FieldSet(const vespalib::string & name) : _name(name), _fields() {} - FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {} - const vespalib::string & getName() const { return _name; } - const Fields & getFields() const { return _fields; } + using Fields = std::set<vespalib::string>; + FieldSet() = default; + explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {} + FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {} + FieldSet(const FieldSet&) = default; + FieldSet& operator=(const FieldSet&) = default; + FieldSet(FieldSet&&) noexcept = default; + FieldSet& operator=(FieldSet&&) noexcept = default; + + const vespalib::string & getName() const noexcept { return _name; } + const Fields & getFields() const noexcept { return _fields; } FieldSet & add(vespalib::string & field) { _fields.insert(field); return *this; @@ -39,28 +46,31 @@ public: vespalib::string _name; Fields _fields; }; - typedef std::map<vespalib::string, FieldSet> FieldSetMap; + using FieldSetMap = std::map<vespalib::string, FieldSet>; + using ImportedFieldNames = vespalib::hash_set<vespalib::string>; + std::vector<const DocumentType *> _inheritedTypes; - StructDataType::SP _ownedFields; - const StructDataType* _fields; - FieldSetMap _fieldSets; + StructDataType::SP _ownedFields; + const StructDataType* _fields; + FieldSetMap _fieldSets; + ImportedFieldNames _imported_field_names; public: - typedef std::unique_ptr<DocumentType> UP; - typedef std::shared_ptr<DocumentType> SP; + using UP = std::unique_ptr<DocumentType>; + using SP = std::shared_ptr<DocumentType>; DocumentType(); DocumentType(vespalib::stringref name, int32_t id); DocumentType(vespalib::stringref name, int32_t id, const StructDataType& fields); - DocumentType(vespalib::stringref name); + explicit DocumentType(vespalib::stringref name); DocumentType(vespalib::stringref name, const StructDataType& fields); - ~DocumentType(); + ~DocumentType() override; - const StructDataType& getFieldsType() const { return *_fields; } + const StructDataType& getFieldsType() const noexcept { return *_fields; } void addField(const Field&); @@ -89,9 +99,16 @@ public: Field::Set getFieldSet() const override; DocumentType* clone() const override; - DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields); + DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields); const FieldSet * getFieldSet(const vespalib::string & name) const; + const ImportedFieldNames& imported_field_names() const noexcept { + return _imported_field_names; + } + bool has_imported_field_name(const vespalib::string& name) const noexcept; + // Ideally the type would be immutable, but this is how it's built today. + void add_imported_field_name(const vespalib::string& name); + DECLARE_IDENTIFIABLE(DocumentType); }; diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index 7c308202e3b..ed17d22da59 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -118,7 +118,7 @@ StructDataType::addInheritedField(const Field& field) FieldValue::UP StructDataType::createFieldValue() const { - return FieldValue::UP(new StructFieldValue(*this)); + return std::make_unique<StructFieldValue>(*this); } const Field& diff --git a/document/src/vespa/document/fieldset/fieldsets.cpp b/document/src/vespa/document/fieldset/fieldsets.cpp index cb162017777..1f9ea32273b 100644 --- a/document/src/vespa/document/fieldset/fieldsets.cpp +++ b/document/src/vespa/document/fieldset/fieldsets.cpp @@ -149,7 +149,7 @@ Document::UP FieldSet::createDocumentSubsetCopy(const Document& src, const FieldSet& fields) { - Document::UP ret(new Document(src.getType(), src.getId())); + auto ret = std::make_unique<Document>(src.getType(), src.getId()); copyFields(*ret, src, fields); return ret; } diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 29414c901f8..fb79b516b90 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document.h" +#include "structuredcache.h" #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/repo/documenttyperepo.h> @@ -8,12 +9,13 @@ #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/xmlstream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <cassert> #include <sstream> -#include <limits> using vespalib::nbostream; using vespalib::make_string; @@ -24,10 +26,6 @@ using namespace vespalib::xml; namespace document { namespace { -bool isLegalVersion(uint16_t version) { - return (6 <= version) && (version <= 8); -} - void documentTypeError(vespalib::stringref name) __attribute__((noinline)); void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline)); @@ -74,17 +72,25 @@ Document::Document() : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(getType().getFieldsType()), + _backingBuffer(), _lastModified(0) { _fields.setDocumentType(getType()); } -Document::Document(const Document& other) = default; +Document::Document(const Document& rhs) + : StructuredFieldValue(rhs), + _id(rhs._id), + _fields(rhs._fields), + _backingBuffer(), + _lastModified(rhs._lastModified) +{} Document::Document(const DataType &type, DocumentId documentId) : StructuredFieldValue(verifyDocumentType(&type)), _id(std::move(documentId)), _fields(getType().getFieldsType()), + _backingBuffer(), _lastModified(0) { _fields.setDocumentType(getType()); @@ -93,65 +99,69 @@ Document::Document(const DataType &type, DocumentId documentId) } } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserialize(repo, buffer); -} - void Document::setRepo(const DocumentTypeRepo& repo) { _fields.setRepo(repo); } -Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), +Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is) + : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), + _backingBuffer(), _lastModified(0) { deserialize(repo, is); } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), +Document::Document(const DocumentTypeRepo& repo, vespalib::DataBuffer && backingBuffer) + : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), + _backingBuffer(), _lastModified(0) { - if (!includeContent) { - const DocumentType *newDocType = deserializeDocHeaderAndType(repo, buffer, _id, static_cast<const DocumentType*>(anticipatedType)); - if (newDocType) { - setType(*newDocType); - } + if (backingBuffer.referencesExternalData()) { + vespalib::nbostream is(backingBuffer.getData(), backingBuffer.getDataLen()); + deserialize(repo, is); } else { - deserialize(repo, buffer); + vespalib::nbostream_longlivedbuf is(backingBuffer.getData(), backingBuffer.getDataLen()); + deserialize(repo, is); + _backingBuffer = std::make_unique<vespalib::DataBuffer>(std::move(backingBuffer)); } } +Document::Document(Document &&) noexcept = default; +Document::~Document() noexcept = default; -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserializeHeader(repo, header); - deserializeBody(repo, body); +Document & +Document::operator =(Document &&rhs) noexcept { + assert( ! _cache && ! rhs._cache); + _id = std::move(rhs._id); + _fields = std::move(rhs._fields); + _backingBuffer = std::move(rhs._backingBuffer); + _lastModified = rhs._lastModified; + StructuredFieldValue::operator=(std::move(rhs)); + return *this; } -Document::~Document() = default; +Document & +Document::operator =(const Document &rhs) { + if (this == &rhs) return *this; + assert( ! _cache && ! rhs._cache); + _id = rhs._id; + _fields = rhs._fields; + _lastModified = rhs._lastModified; + StructuredFieldValue::operator=(rhs); + _backingBuffer.reset(); + return *this; +} const DocumentType& Document::getType() const { return static_cast<const DocumentType &>(StructuredFieldValue::getType()); } -Document& Document::operator=(const Document& doc) = default; - void Document::clear() { @@ -170,37 +180,14 @@ Document::hasChanged() const return _fields.hasChanged(); } -DocumentId -Document::getIdFromSerialized(ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - deserializeDocHeader(buf, retVal); - buf.setPos(position); - - return retVal; -} - -const DocumentType * -Document::getDocTypeFromSerialized(const DocumentTypeRepo& repo, ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - const DocumentType *docType(deserializeDocHeaderAndType(repo, buf, retVal, nullptr)); - buf.setPos(position); - - return docType; -} - FieldValue& Document::assign(const FieldValue& value) { /// \todo TODO (was warning): This type checking doesnt work with the way assign is used. // if (*value.getDataType() == *_type) { auto & other(dynamic_cast<const Document&>(value)); - return operator=(other); + *this = Document(other); + return *this; // } // return FieldValue::assign(value); // Generates exception } @@ -274,118 +261,9 @@ Document::calculateChecksum() const return calculator.checksum() ^ _fields.calculateChecksum(); } -const DocumentType * -Document::deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, DocumentId& id, - const DocumentType * docType) -{ - deserializeDocHeader(buffer, id); - - vespalib::stringref docTypeName(buffer.getBufferAtPos()); - buffer.incPos(docTypeName.size() + 1); // Skip 0-byte too - { - int16_t docTypeVersion; // version not supported anymore - buffer.getShortNetwork(docTypeVersion); - } - const DocumentType *docTypeNew = nullptr; - - if (! ((docType != nullptr) && (docType->getName() == docTypeName))) { - docTypeNew = repo.getDocumentType(docTypeName); - if (!docTypeNew) { - throw DocumentTypeNotFoundException(docTypeName, VESPA_STRLOC); - } - } - return docTypeNew; -} - -namespace { -[[noreturn]] void versionError(uint16_t version) __attribute__((noinline)); -[[noreturn]] void mainDocumentError(int64_t len) __attribute__((noinline)); -[[noreturn]] void notEnoughDocumentError(int32_t len, int64_t remaining) __attribute__((noinline)); - -void versionError(uint16_t version) { - throw DeserializeException(make_string( "Unrecognized serialization version %d", version), VESPA_STRLOC); -} - -void mainDocumentError(int64_t len) { - throw DeserializeException(make_string( - "Document lengths past %i is not supported. Corrupt data said length is %" PRId64 " bytes", - std::numeric_limits<int>::max(), len), VESPA_STRLOC); -} - -void notEnoughDocumentError(int32_t len, int64_t remaining) { - throw DeserializeException(make_string( "Buffer said document length is %d bytes, but only %" PRId64 " bytes remain in buffer", len, remaining)); -} - -} - -void -Document::deserializeDocHeader(ByteBuffer& buffer, DocumentId& id) { - int16_t version; - int32_t len; - buffer.getShortNetwork(version); - - if ( ! isLegalVersion(version) ) { - versionError(version); - } else if (version < 7) { - int64_t tmpLen = 0; - buffer.getInt2_4_8Bytes(tmpLen); - if (tmpLen > std::numeric_limits<int>::max()) { - mainDocumentError(tmpLen); - } else { - len = static_cast<int32_t>(tmpLen) - - ByteBuffer::getSerializedSize2_4_8Bytes(tmpLen) - - sizeof(uint16_t); - } - } else { - buffer.getIntNetwork(len); - } - - if (len > (long)buffer.getRemaining()) { - notEnoughDocumentError(len, buffer.getRemaining()); - } else { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - id = DocumentId(stream); - buffer.incPos(stream.rp()); - unsigned char contentByte; - buffer.getByte(contentByte); - } -} - -void Document::serializeHeader(ByteBuffer& buffer) const { - nbostream stream; - serializeHeader(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - void Document::serializeHeader(nbostream& stream) const { VespaDocumentSerializer serializer(stream); - serializer.write(*this, WITHOUT_BODY); -} - -void Document::serializeBody(ByteBuffer& buffer) const { - nbostream stream; - serializeBody(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -bool Document::hasBodyField() const { - for (document::StructuredFieldValue::const_iterator it(getFields().begin()), mt(getFields().end()); - it != mt; - ++it) - { - if ( ! it.field().isHeaderField() ) { - return true; - } - } - return false; -} - -void Document::serializeBody(nbostream& stream) const { - if (hasBodyField()) { - VespaDocumentSerializer serializer(stream); - serializer.write(_fields, BodyFields()); - } + serializer.write(*this); } void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os) { @@ -397,38 +275,19 @@ void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & o } } -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& data) { - nbostream stream(data.getBufferAtPos(), data.getRemaining()); - deserialize(repo, stream); - data.incPos(data.getRemaining() - stream.size()); -} - -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body) { +void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & header, vespalib::nbostream & body) { deserializeHeader(repo, header); deserializeBody(repo, body); } -void Document::deserializeHeader(const DocumentTypeRepo& repo, - ByteBuffer& header) { - nbostream stream(header.getBufferAtPos(), header.getRemaining()); +void Document::deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { VespaDocumentDeserializer deserializer(repo, stream, 0); deserializer.read(*this); - header.incPos(header.getRemaining() - stream.size()); } -void Document::deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body) { - nbostream body_stream(body.getBufferAtPos(), body.getRemaining()); - VespaDocumentDeserializer - body_deserializer(repo, body_stream, getFields().getVersion()); - body_deserializer.readStructNoReset(getFields()); - body.incPos(body.getRemaining() - body_stream.size()); -} - -size_t -Document::getSerializedSize() const -{ - // Temporary non-optimal (but guaranteed correct) implementation. - return serialize()->getLength(); +void Document::deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { + VespaDocumentDeserializer deserializer(repo, stream, getFields().getVersion()); + deserializer.readStructNoReset(getFields()); } StructuredFieldValue::StructuredIterator::UP @@ -437,4 +296,20 @@ Document::getIterator(const Field* first) const return _fields.getIterator(first); } +void +Document::beginTransaction() { + _cache = std::make_unique<StructuredCache>(); +} +void +Document::commitTransaction() { + for (auto & e : *_cache) { + if (e.second.status == fieldvalue::ModificationStatus::REMOVED) { + removeFieldValue(e.first); + } else if (e.second.status == fieldvalue::ModificationStatus::MODIFIED) { + setFieldValue(e.first, std::move(e.second.value)); + } + } + _cache.reset(); +} + } // document diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h index 3e1ec0da3e6..cb27ca5f338 100644 --- a/document/src/vespa/document/fieldvalue/document.h +++ b/document/src/vespa/document/fieldvalue/document.h @@ -19,18 +19,24 @@ #include <vespa/document/base/documentid.h> #include <vespa/document/base/field.h> +namespace vespalib { class DataBuffer; } namespace document { +class TransactionGuard; + class Document : public StructuredFieldValue { private: DocumentId _id; StructFieldValue _fields; + std::unique_ptr<StructuredCache> _cache; + std::unique_ptr<vespalib::DataBuffer> _backingBuffer; - // To avoid having to return another container object out of docblocks - // the meta data has been added to document. This will not be serialized - // with the document and really doesn't belong here! + // To avoid having to return another container object out of docblocks + // the meta data has been added to document. This will not be serialized + // with the document and really doesn't belong here! int64_t _lastModified; + public: typedef std::unique_ptr<Document> UP; typedef std::shared_ptr<Document> SP; @@ -41,21 +47,17 @@ public: Document(); Document(const Document&); + Document(Document &&) noexcept; + Document & operator =(const Document &); + Document & operator =(Document &&) noexcept; Document(const DataType &, DocumentId id); - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType = nullptr); - Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream, const DataType *anticipatedType = nullptr); - /** - Constructor to deserialize only document and type from a buffer. Only relevant if includeContent is false. - */ - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType); - Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType = nullptr); - ~Document() override; + Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream); + Document(const DocumentTypeRepo& repo, vespalib::DataBuffer && buffer); + ~Document() noexcept override; void setRepo(const DocumentTypeRepo & repo); const DocumentTypeRepo * getRepo() const { return _fields.getRepo(); } - Document& operator=(const Document&); - void accept(FieldValueVisitor &visitor) override { visitor.visit(*this); } void accept(ConstFieldValueVisitor &visitor) const override { visitor.visit(*this); } @@ -86,21 +88,6 @@ public: bool hasChanged() const override; - /** - * Returns a pointer to the Id of a serialized document, without performing - * the deserialization. buffer must point to the start position of the - * serialization. If the buffer doesn't have enough data remaining to have - * a legal Id in it, method returns NULL. - */ - static DocumentId getIdFromSerialized(ByteBuffer&); - - /** - * Returns a pointer to the document type of a serialized header, without - * performing the deserialization. Buffer must point to the start position - * of the serialization. - */ - static const DocumentType *getDocTypeFromSerialized(const DocumentTypeRepo&, ByteBuffer&); - // FieldValue implementation. FieldValue& assign(const FieldValue&) override; int compare(const FieldValue& other) const override; @@ -108,22 +95,12 @@ public: void printXml(XmlOutputStream& out) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - // Specialized serialization functions - void serializeHeader(ByteBuffer& buffer) const; + // Specialized serialization functions, Only used for testing legacy stuff void serializeHeader(vespalib::nbostream& stream) const; - void serializeBody(ByteBuffer& buffer) const; - void serializeBody(vespalib::nbostream& stream) const; - - /** Deserialize document contained in given bytebuffer. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& data); void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os); /** Deserialize document contained in given bytebuffers. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& body, ByteBuffer& header); - void deserializeHeader(const DocumentTypeRepo& repo, ByteBuffer& header); - void deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body); - - size_t getSerializedSize() const; + void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & body, vespalib::nbostream & header); /** Undo fieldvalue's toXml override for document. */ std::string toXml() const { return toXml(""); } @@ -137,18 +114,30 @@ public: void setFieldValue(const Field& field, FieldValue::UP data) override; private: - bool hasBodyField() const; + friend TransactionGuard; + void beginTransaction(); + void commitTransaction(); + void deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & header); + void deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & body); bool hasFieldValue(const Field& field) const override { return _fields.hasValue(field); } void removeFieldValue(const Field& field) override { _fields.remove(field); } FieldValue::UP getFieldValue(const Field& field) const override { return _fields.getValue(field); } bool getFieldValue(const Field& field, FieldValue& value) const override { return _fields.getValue(field, value); } StructuredIterator::UP getIterator(const Field* first) const override; + StructuredCache * getCache() const override { return _cache.get(); } +}; - static void deserializeDocHeader(ByteBuffer& buffer, DocumentId& id); - static const DocumentType *deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, - DocumentId& id, const DocumentType * docType); +class TransactionGuard { +public: + TransactionGuard(Document & value) + : _value(value) + { + _value.beginTransaction(); + } + ~TransactionGuard() { _value.commitTransaction(); } +private: + Document & _value; }; } // document diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 999747688eb..7737d03b45a 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -35,22 +35,11 @@ void FieldValue::serialize(nbostream &stream) const { serializer.write(*this); } -void FieldValue::serialize(ByteBuffer& buffer) const { +nbostream +FieldValue::serialize() const { nbostream stream; serialize(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -std::unique_ptr<ByteBuffer> FieldValue::serialize() const { - nbostream stream; - serialize(stream); - - nbostream::Buffer buf; - stream.swap(buf); - size_t sz = buf.size(); - auto bb = std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); - bb->setPos(sz); - return bb; + return stream; } size_t @@ -58,7 +47,7 @@ FieldValue::hash() const { vespalib::nbostream os; serialize(os); - return vespalib::hashValue(os.c_str(), os.size()) ; + return vespalib::hashValue(os.data(), os.size()) ; } bool diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index a152f74cc09..66d5eaa8c58 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -19,15 +19,11 @@ #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/util/polymorphicarraybase.h> -namespace vespalib { - class nbostream; -} +namespace vespalib { class nbostream; } namespace document { -namespace fieldvalue { - class IteratorHandler; -} +namespace fieldvalue { class IteratorHandler; } class ByteBuffer; class DataType; @@ -37,6 +33,8 @@ class FieldValue : public vespalib::Identifiable protected: FieldValue(const FieldValue&) = default; FieldValue& operator=(const FieldValue&) = default; + FieldValue(FieldValue &&) = default; + FieldValue& operator=(FieldValue &&) = default; static std::unique_ptr<vespalib::IArrayBase> createArray(const DataType & baseType); public: @@ -73,8 +71,7 @@ public: virtual bool isA(const FieldValue& other) const; void serialize(vespalib::nbostream &stream) const; - void serialize(ByteBuffer& buffer) const; - std::unique_ptr<ByteBuffer> serialize() const; + vespalib::nbostream serialize() const; /** * Compares this fieldvalue with another fieldvalue. diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 5dfd8eff891..84fe7ba34a6 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -17,7 +17,7 @@ namespace document { namespace serializablearray { -using BufferMapT = vespalib::hash_map<int, ByteBuffer::UP>; +using BufferMapT = vespalib::hash_map<int, ByteBuffer>; class BufferMap : public BufferMapT { public: @@ -26,83 +26,105 @@ public: } -SerializableArray::Statistics SerializableArray::_stats; - -SerializableArray::SerializableArray() - : _serializedCompression(CompressionConfig::NONE), - _uncompressedLength(0) -{ +void +SerializableArray::set(EntryMap entries, ByteBuffer buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length) +{ + _entries = std::move(entries); + if (CompressionConfig::isCompressed(comp_type)) { + _unlikely = std::make_unique<RarelyUsedBuffers>(); + _unlikely->_compSerData = std::move(buffer); + _unlikely->_serializedCompression = comp_type; + _unlikely->_uncompressedLength = uncompressed_length; + _uncompSerData = ByteBuffer(); + } else { + _uncompSerData = std::move(buffer); + _unlikely.reset(); + } } -serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { +SerializableArray::SerializableArray(SerializableArray &&) noexcept = default; +SerializableArray& SerializableArray::operator=(SerializableArray &&) noexcept = default; +SerializableArray::~SerializableArray() = default; + +namespace { + +template <typename T> +T & +ensure(std::unique_ptr<T> &owned) { if (!owned) { - owned = std::make_unique<serializablearray::BufferMap>(); + owned = std::make_unique<T>(); } return *owned; } -SerializableArray::SerializableArray(const SerializableArray& other) - : Cloneable(), - _entries(other._entries), - _owned(), - _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : NULL), - _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : NULL), - _serializedCompression(other._serializedCompression), - _uncompressedLength(other._uncompressedLength) +} + +SerializableArray::RarelyUsedBuffers::RarelyUsedBuffers() + : _owned(), + _compSerData(nullptr, 0), + _serializedCompression(CompressionConfig::NONE), + _uncompressedLength(0) +{ } +SerializableArray::RarelyUsedBuffers::~RarelyUsedBuffers() = default; + +SerializableArray::RarelyUsedBuffers::RarelyUsedBuffers(const RarelyUsedBuffers & rhs) + : _owned(), + _compSerData(rhs._compSerData), + _serializedCompression(rhs._serializedCompression), + _uncompressedLength(rhs._uncompressedLength) +{ } + +SerializableArray::SerializableArray(const SerializableArray& rhs) + : _entries(rhs._entries), + _uncompSerData(rhs._uncompSerData), + _unlikely(rhs._unlikely ? new RarelyUsedBuffers(*rhs._unlikely) : nullptr) { for (size_t i(0); i < _entries.size(); i++) { Entry & e(_entries[i]); if (e.hasBuffer()) { // Pointing to a buffer in the _owned structure. - ByteBuffer::UP buf(ByteBuffer::copyBuffer(e.getBuffer(_uncompSerData.get()), e.size())); - e.setBuffer(buf->getBuffer()); - ensure(_owned)[e.id()] = std::move(buf); + ByteBuffer buf(ByteBuffer::copyBuffer(e.getBuffer(&_uncompSerData), e.size())); + e.setBuffer(buf.getBuffer()); + ensure(_unlikely->_owned)[e.id()] = std::move(buf); } else { // If not it is relative to the buffer _uncompSerData, and hence it is valid as is. } } - if (_uncompSerData.get()) { - LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); - } } -void -SerializableArray::swap(SerializableArray& other) +SerializableArray & +SerializableArray::operator=(const SerializableArray &rhs) { - _entries.swap(other._entries); - _owned.swap(other._owned); - std::swap(_uncompSerData, other._uncompSerData); - std::swap(_compSerData, other._compSerData); - std::swap(_serializedCompression, other._serializedCompression); - std::swap(_uncompressedLength, other._uncompressedLength); + if (this != &rhs) { + *this = SerializableArray(rhs); + } + return *this; } void SerializableArray::clear() { _entries.clear(); - _uncompSerData.reset(); - _compSerData.reset(); - _serializedCompression = CompressionConfig::NONE; - _uncompressedLength = 0; -} - -SerializableArray::~SerializableArray() -{ + _uncompSerData = ByteBuffer(nullptr, 0); + _unlikely.reset(); } void SerializableArray::invalidate() { - _compSerData.reset(); + if (_unlikely) { + _unlikely->_compSerData = ByteBuffer(nullptr, 0);; + } } void -SerializableArray::set(int id, ByteBuffer::UP buffer) +SerializableArray::set(int id, ByteBuffer buffer) { maybeDecompress(); - Entry e(id, buffer->getRemaining(), buffer->getBuffer()); - ensure(_owned)[id] = std::move(buffer); - EntryMap::iterator it = find(id); + Entry e(id, buffer.getRemaining(), buffer.getBuffer()); + assert(buffer.getRemaining() < 0x80000000ul); + ensure(ensure(_unlikely)._owned)[id] = std::move(buffer); + auto it = find(id); if (it == _entries.end()) { _entries.push_back(e); } else { @@ -113,7 +135,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer) void SerializableArray::set(int id, const char* value, int len) { - set(id, std::unique_ptr<ByteBuffer>(ByteBuffer::copyBuffer(value,len))); + set(id, ByteBuffer::copyBuffer(value,len)); } SerializableArray::EntryMap::const_iterator @@ -139,11 +161,11 @@ SerializableArray::get(int id) const { vespalib::ConstBufferRef buf; if ( !maybeDecompressAndCatch() ) { - EntryMap::const_iterator found = find(id); + auto found = find(id); if (found != _entries.end()) { const Entry& entry = *found; - buf = vespalib::ConstBufferRef(entry.getBuffer(_uncompSerData.get()), entry.size()); + buf = vespalib::ConstBufferRef(entry.getBuffer(&_uncompSerData), entry.size()); } } else { // should we clear all or what? @@ -168,11 +190,11 @@ void SerializableArray::clear(int id) { maybeDecompress(); - EntryMap::iterator it = find(id); + auto it = find(id); if (it != _entries.end()) { _entries.erase(it); - if (_owned) { - _owned->erase(id); + if (_unlikely && _unlikely->_owned) { + _unlikely->_owned->erase(id); } invalidate(); } @@ -184,64 +206,42 @@ SerializableArray::deCompress() // throw (DeserializeException) using vespalib::compression::decompress; // will only do this once - LOG_ASSERT(_compSerData); - LOG_ASSERT(!_uncompSerData); - - if (_serializedCompression == CompressionConfig::NONE || - _serializedCompression == CompressionConfig::UNCOMPRESSABLE) - { - _uncompSerData = std::move(_compSerData); - LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); - } else { - ByteBuffer::UP newSerialization(new ByteBuffer(_uncompressedLength)); - vespalib::DataBuffer unCompressed(newSerialization->getBuffer(), newSerialization->getLength()); - unCompressed.clear(); - try { - decompress(_serializedCompression, - _uncompressedLength, - vespalib::ConstBufferRef(_compSerData->getBufferAtPos(), _compSerData->getRemaining()), - unCompressed, - false); - } catch (const std::runtime_error & e) { - throw DeserializeException( - make_string( "Document was compressed with code unknown code %d", _serializedCompression), - VESPA_STRLOC); - } + assert(_unlikely && (_unlikely->_compSerData.getRemaining() != 0)); + assert(_uncompSerData.getRemaining() == 0); + assert(CompressionConfig::isCompressed(_unlikely->_serializedCompression)); + uint32_t uncompressedLength = _unlikely->_uncompressedLength; - if (unCompressed.getDataLen() != (size_t)_uncompressedLength) { - throw DeserializeException( - make_string("Did not decompress to the expected length: had %zu, wanted %d, got %zu", - _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), - VESPA_STRLOC); - } - assert(newSerialization->getBuffer() == unCompressed.getData()); - newSerialization->setLimit(_uncompressedLength); - _uncompSerData = std::move(newSerialization); - LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); + ByteBuffer newSerialization(vespalib::alloc::Alloc::alloc(uncompressedLength), uncompressedLength); + vespalib::DataBuffer unCompressed(newSerialization.getBuffer(), newSerialization.getLength()); + unCompressed.clear(); + try { + decompress(_unlikely->_serializedCompression, + uncompressedLength, + vespalib::ConstBufferRef(_unlikely->_compSerData.getBufferAtPos(), _unlikely->_compSerData.getRemaining()), + unCompressed, + false); + } catch (const std::runtime_error & e) { + throw DeserializeException( + make_string( "Document was compressed with code unknown code %d", _unlikely->_serializedCompression), + VESPA_STRLOC); } -} - -void SerializableArray::assign(EntryMap & entries, - ByteBuffer::UP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length) -{ - _serializedCompression = comp_type; - _entries.clear(); - _entries.swap(entries); - if (CompressionConfig::isCompressed(_serializedCompression)) { - _compSerData.reset(buffer.release()); - _uncompressedLength = uncompressed_length; - } else { - _uncompressedLength = buffer->getRemaining(); - _uncompSerData.reset(buffer.release()); + if (unCompressed.getDataLen() != (size_t)uncompressedLength) { + throw DeserializeException( + make_string("Did not decompress to the expected length: had %u, wanted %d, got %zu", + _unlikely->_compSerData.getRemaining(), uncompressedLength, unCompressed.getDataLen()), + VESPA_STRLOC); } + assert(newSerialization.getBuffer() == unCompressed.getData()); + _uncompSerData = std::move(newSerialization); + LOG_ASSERT(uncompressedLength == _uncompSerData.getRemaining()); } vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { - return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); + return _unlikely + ? CompressionInfo(_unlikely->_uncompressedLength, _unlikely->_compSerData.getRemaining()) + : CompressionInfo(_uncompSerData.getRemaining(), CompressionConfig::NONE); } const char * diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2f7d65938aa..11186593e76 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -17,44 +17,24 @@ #pragma once #include <vespa/vespalib/util/compressionconfig.h> -#include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> +#include <vespa/document/util/bytebuffer.h> #include <vector> #define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) namespace document { -class SerializableArrayIterator; class ByteBuffer; namespace serializablearray { class BufferMap; } -class SerializableArray : public vespalib::Cloneable +class SerializableArray { public: - // Counts set during serialization, in order to provide metrics for how - // often we use cached version, and how often we compress. - struct Statistics { - uint64_t _usedCachedSerializationCount; - uint64_t _compressedDocumentCount; - uint64_t _compressionDidntHelpCount; - uint64_t _uncompressableCount; - uint64_t _serializedUncompressed; - uint64_t _inputWronglySerialized; - - Statistics() - : _usedCachedSerializationCount(0), - _compressedDocumentCount(0), - _compressionDidntHelpCount(0), - _uncompressableCount(0), - _serializedUncompressed(0), - _inputWronglySerialized(0) {} - }; - /** * Contains the id of a field, the size and a buffer reference that is either * a relative offset to a common buffer, or the buffer itself it it is not. @@ -98,23 +78,21 @@ public: static const uint32_t ReservedId = 100; static const uint32_t ReservedIdUpper = 128; - -private: - static Statistics _stats; - -public: - static Statistics& getStatistics() { return _stats; } using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; using CompressionConfig = vespalib::compression::CompressionConfig; using CompressionInfo = vespalib::compression::CompressionInfo; - SerializableArray(); - virtual ~SerializableArray(); - - void swap(SerializableArray& other); + SerializableArray() = default; + SerializableArray(const SerializableArray&); + SerializableArray& operator=(const SerializableArray&); + SerializableArray(SerializableArray &&) noexcept; + SerializableArray& operator=(SerializableArray &&) noexcept; + ~SerializableArray(); + void set(EntryMap entries, ByteBuffer buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length); /** * Stores a value in the array. * @@ -125,7 +103,7 @@ public: void set(int id, const char* value, int len); /** Stores a value in the array. */ - void set(int id, ByteBufferUP buffer); + void set(int id, ByteBuffer buffer); /** * Gets a value from the array. This is the faster version of the above. @@ -141,9 +119,6 @@ public: /** @return Returns true if the given ID is Set in the array. */ bool has(int id) const; - /** @return Number of elements in array */ - bool hasAnyElems() const { return !_entries.empty(); } - /** * clears an attribute. * @@ -154,34 +129,23 @@ public: /** Deletes all stored attributes. */ void clear(); - CompressionConfig::Type getCompression() const { return _serializedCompression; } + CompressionConfig::Type getCompression() const { + return _unlikely ? _unlikely->_serializedCompression : CompressionConfig::NONE; + } CompressionInfo getCompressionInfo() const; - /** - * Sets the serialized data that is the basis for this object's - * content. This is used by deserialization. Any existing entries - * are cleared. - */ - void assign(EntryMap &entries, - ByteBufferUP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length); - bool empty() const { return _entries.empty(); } const ByteBuffer* getSerializedBuffer() const { - return CompressionConfig::isCompressed(_serializedCompression) - ? _compSerData.get() - : _uncompSerData.get(); + return CompressionConfig::isCompressed(getCompression()) + ? &_unlikely->_compSerData + : &_uncompSerData; } - SerializableArray* clone() const override { return new SerializableArray(*this); } - SerializableArray(const SerializableArray&); // Public only for test - SerializableArray& operator=(const SerializableArray&) = delete; const EntryMap & getEntries() const { return _entries; } private: bool shouldDecompress() const { - return _compSerData.get() && !_uncompSerData.get(); + return _unlikely && (_unlikely->_compSerData.getRemaining() != 0) && (_uncompSerData.getBuffer() == 0); } bool maybeDecompressAndCatch() const { if ( shouldDecompress() ) { @@ -198,17 +162,22 @@ private: } void deCompress(); // throw (DeserializeException); + struct RarelyUsedBuffers { + /** The buffers we own. */ + RarelyUsedBuffers(); + RarelyUsedBuffers(const RarelyUsedBuffers &); + ~RarelyUsedBuffers(); + std::unique_ptr<serializablearray::BufferMap> _owned; + ByteBuffer _compSerData; + CompressionConfig::Type _serializedCompression; + uint32_t _uncompressedLength; + }; /** Contains the stored attributes, with reference to the real data.. */ - EntryMap _entries; - /** The buffers we own. */ - std::unique_ptr<serializablearray::BufferMap> _owned; - + EntryMap _entries; /** Data we deserialized from, if applicable. */ - ByteBufferUP _uncompSerData; - ByteBufferUP _compSerData; - CompressionConfig::Type _serializedCompression; + ByteBuffer _uncompSerData; + std::unique_ptr<RarelyUsedBuffers> _unlikely; - uint32_t _uncompressedLength; VESPA_DLL_LOCAL void invalidate(); VESPA_DLL_LOCAL EntryMap::const_iterator find(int id) const; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 87c8d09648c..bc2dd2a059a 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -32,6 +32,7 @@ IMPLEMENT_IDENTIFIABLE_ABSTRACT(StructFieldValue, StructuredFieldValue); StructFieldValue::StructFieldValue(const DataType &type) : StructuredFieldValue(type), + _fields(), _repo(nullptr), _doc_type(nullptr), _version(Document::getNewestSerializationVersion()), @@ -42,22 +43,7 @@ StructFieldValue::StructFieldValue(const DataType &type) StructFieldValue::StructFieldValue(const StructFieldValue & rhs) = default; StructFieldValue & StructFieldValue::operator = (const StructFieldValue & rhs) = default; -StructFieldValue::~StructFieldValue() = default; - -StructFieldValue::Chunks::~Chunks() = default; - -void -StructFieldValue::Chunks::push_back(SerializableArray::UP item) { - assert(_sz < 2); - _chunks[_sz++].reset(item.release()); -} - -void -StructFieldValue::Chunks::clear() { - _chunks[0].reset(); - _chunks[1].reset(); - _sz = 0; -} +StructFieldValue::~StructFieldValue() noexcept = default; const StructDataType & StructFieldValue::getStructType() const { @@ -73,7 +59,7 @@ void StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fm, - ByteBuffer::UP buffer, + ByteBuffer buffer, CompressionConfig::Type comp_type, int32_t uncompressed_length) { @@ -81,20 +67,15 @@ StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, _doc_type = &repo.getDocumentType(); _version = version; - _chunks.push_back(std::make_unique<SerializableArray>()); - _chunks.back().assign(fm, std::move(buffer), comp_type, uncompressed_length); + _fields.set(std::move(fm), std::move(buffer), comp_type, uncompressed_length); _hasChanged = false; } -bool StructFieldValue::serializeField(int field_id, uint16_t version, - FieldValueWriter &writer) const { +bool StructFieldValue::serializeField(int field_id, uint16_t version, FieldValueWriter &writer) const { if (version == _version) { - for (int i = _chunks.size() - 1; i >= 0; --i) { - vespalib::ConstBufferRef buf = _chunks[i].get(field_id); - if ( buf.size() != 0) { - writer.writeSerializedData(buf.data(), buf.size()); - break; - } + vespalib::ConstBufferRef buf = _fields.get(field_id); + if ( buf.size() != 0) { + writer.writeSerializedData(buf.data(), buf.size()); } return true; @@ -111,33 +92,21 @@ bool StructFieldValue::serializeField(int field_id, uint16_t version, void StructFieldValue::getRawFieldIds(vector<int> &raw_ids) const { raw_ids.clear(); - - size_t count(0); - for (uint32_t i = 0; i < _chunks.size(); ++i) { - count += _chunks[i].getEntries().size(); - } - raw_ids.reserve(count); - for (uint32_t i = 0; i < _chunks.size(); ++i) { - const SerializableArray::EntryMap & entries = _chunks[i].getEntries(); - for (const SerializableArray::Entry & entry : entries) { - raw_ids.emplace_back(entry.id()); - } + raw_ids.reserve(_fields.getEntries().size()); + for (const SerializableArray::Entry & entry : _fields.getEntries()) { + raw_ids.emplace_back(entry.id()); } sort(raw_ids.begin(), raw_ids.end()); raw_ids.erase(unique(raw_ids.begin(), raw_ids.end()), raw_ids.end()); } void -StructFieldValue::getRawFieldIds(vector<int> &raw_ids, - const FieldSet& fieldSet) const { +StructFieldValue::getRawFieldIds(vector<int> &raw_ids,const FieldSet& fieldSet) const { raw_ids.clear(); - for (uint32_t i = 0; i < _chunks.size(); ++i) { - const SerializableArray::EntryMap & entries = _chunks[i].getEntries(); - for (const SerializableArray::Entry & entry : entries) { - if (fieldSet.contains(getStructType().getField(entry.id()))) { - raw_ids.push_back(entry.id()); - } + for (const SerializableArray::Entry & entry : _fields.getEntries()) { + if (fieldSet.contains(getStructType().getField(entry.id()))) { + raw_ids.emplace_back(entry.id()); } } sort(raw_ids.begin(), raw_ids.end()); @@ -178,19 +147,17 @@ StructFieldValue::getFieldValue(const Field& field) const { int fieldId = field.getId(); - for (int i = _chunks.size() - 1; i >= 0; --i) { - vespalib::ConstBufferRef buf = _chunks[i].get(fieldId); - if (buf.size() != 0) { - nbostream stream(buf.c_str(), buf.size()); - FieldValue::UP value(field.getDataType().createFieldValue()); - if ((_repo == NULL) && (_doc_type != NULL)) { - std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); - createFV(*value, *tmpRepo, stream, *_doc_type, _version); - } else { - createFV(*value, *_repo, stream, *_doc_type, _version); - } - return value; + vespalib::ConstBufferRef buf = _fields.get(fieldId); + if (buf.size() != 0) { + nbostream stream(buf.c_str(), buf.size()); + FieldValue::UP value(field.getDataType().createFieldValue()); + if ((_repo == nullptr) && (_doc_type != nullptr)) { + DocumentTypeRepo tmpRepo(*_doc_type); + createFV(*value, tmpRepo, stream, *_doc_type, _version); + } else { + createFV(*value, *_repo, stream, *_doc_type, _version); } + return value; } return FieldValue::UP(); } @@ -198,11 +165,9 @@ StructFieldValue::getFieldValue(const Field& field) const vespalib::ConstBufferRef StructFieldValue::getRawField(uint32_t id) const { - for (int i = _chunks.size() - 1; i >= 0; --i) { - vespalib::ConstBufferRef buf = _chunks[i].get(id); - if (buf.size() > 0) { - return buf; - } + vespalib::ConstBufferRef buf = _fields.get(id); + if (buf.size() > 0) { + return buf; } return vespalib::ConstBufferRef(); @@ -216,9 +181,9 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const vespalib::ConstBufferRef buf = getRawField(fieldId); if (buf.size() > 0) { nbostream_longlivedbuf stream(buf.c_str(), buf.size()); - if ((_repo == NULL) && (_doc_type != NULL)) { - std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); - createFV(value, *tmpRepo, stream, *_doc_type, _version); + if ((_repo == nullptr) && (_doc_type != nullptr)) { + DocumentTypeRepo tmpRepo(*_doc_type); + createFV(value, tmpRepo, stream, *_doc_type, _version); } else { createFV(value, *_repo, stream, *_doc_type, _version); } @@ -230,26 +195,29 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const bool StructFieldValue::hasFieldValue(const Field& field) const { - for (int i = _chunks.size() - 1; i >= 0; --i) { - if (_chunks[i].has(field.getId())) { - return true; - } - } + return _fields.has(field.getId()); +} - return false; +namespace { + +std::unique_ptr<ByteBuffer> +serializeDoc(const FieldValue & fv) { + nbostream stream = fv.serialize(); + nbostream::Buffer buf; + stream.swap(buf); + size_t sz = buf.size(); + return std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); } +} void StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value) { int fieldId = field.getId(); - std::unique_ptr<ByteBuffer> serialized(value->serialize()); - serialized->flip(); - if (_chunks.empty()) { - _chunks.push_back(SerializableArray::UP(new SerializableArray())); - } - _chunks.back().set(fieldId, std::move(serialized)); + std::unique_ptr<ByteBuffer> serialized = serializeDoc(*value); + + _fields.set(fieldId, std::move(*serialized)); _hasChanged = true; } @@ -257,16 +225,14 @@ StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value) void StructFieldValue::removeFieldValue(const Field& field) { - for (uint32_t i = 0; i < _chunks.size(); ++i) { - _chunks[i].clear(field.getId()); - } + _fields.clear(field.getId()); _hasChanged = true; } void StructFieldValue::clear() { - _chunks.clear(); + _fields.clear(); _hasChanged = true; } @@ -274,7 +240,7 @@ StructFieldValue::clear() FieldValue& StructFieldValue::assign(const FieldValue& value) { - const StructFieldValue& other(dynamic_cast<const StructFieldValue&>(value)); + const auto & other(dynamic_cast<const StructFieldValue&>(value)); return operator=(other); } @@ -285,7 +251,7 @@ StructFieldValue::compare(const FieldValue& otherOrg) const if (comp != 0) { return comp; } - const StructFieldValue& other = static_cast<const StructFieldValue&>(otherOrg); + const auto & other = static_cast<const StructFieldValue&>(otherOrg); std::vector<int> a; getRawFieldIds(a); @@ -315,9 +281,9 @@ StructFieldValue::compare(const FieldValue& otherOrg) const uint32_t StructFieldValue::calculateChecksum() const { - ByteBuffer::UP buffer(serialize()); + nbostream buffer(serialize()); vespalib::crc_32_type calculator; - calculator.process_bytes(buffer->getBuffer(), buffer->getPos()); + calculator.process_bytes(buffer.peek(), buffer.size()); return calculator.checksum(); } @@ -365,13 +331,7 @@ StructFieldValue::print(std::ostream& out, bool verbose, bool StructFieldValue::empty() const { - for (uint32_t i = 0; i < _chunks.size(); ++i) { - if (!_chunks[i].empty()) { - return false; - } - } - - return true; + return _fields.empty(); } void @@ -386,7 +346,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { std::vector<int> _ids; std::vector<int>::iterator _cur; - FieldIterator(const StructFieldValue& s) + explicit FieldIterator(const StructFieldValue& s) : _struct(s), _ids(), _cur(_ids.begin()) @@ -412,28 +372,25 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { LOG(debug, "struct data type: %s", _struct.getType().toString(true).c_str()); } } - return 0; + return nullptr; } }; StructuredFieldValue::StructuredIterator::UP StructFieldValue::getIterator(const Field* toFind) const { - StructuredIterator::UP ret; + auto fi = std::make_unique<FieldIterator>(*this); - FieldIterator *fi = new FieldIterator(*this); - ret.reset(fi); - - if (toFind != NULL) { + if (toFind != nullptr) { fi->skipTo(toFind->getId()); } - return ret; + return fi; } void StructFieldValue::setType(const DataType& type) { - clear(); + reset(); StructuredFieldValue::setType(type); } diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index b5b15e9dfce..fc930951df3 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -24,26 +24,8 @@ class StructDataType; class StructFieldValue : public StructuredFieldValue { -public: - class Chunks { - public: - Chunks() : _sz(0) { } - ~Chunks(); - SerializableArray & operator [] (size_t i) { return *_chunks[i]; } - const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; } - VESPA_DLL_LOCAL void push_back(SerializableArray::UP item); - SerializableArray & back() { return *_chunks[_sz-1]; } - const SerializableArray & back() const { return *_chunks[_sz-1]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } - VESPA_DLL_LOCAL void clear(); - private: - SerializableArray::CP _chunks[2]; - size_t _sz; - }; private: - Chunks _chunks; - + SerializableArray _fields; // As we do lazy deserialization, we need these saved const DocumentTypeRepo *_repo; const DocumentType *_doc_type; @@ -57,16 +39,19 @@ public: StructFieldValue(const DataType &type); StructFieldValue(const StructFieldValue & rhs); StructFieldValue & operator = (const StructFieldValue & rhs); - ~StructFieldValue() override; + StructFieldValue(StructFieldValue && rhs) noexcept = default; + StructFieldValue & operator = (StructFieldValue && rhs) noexcept = default; + ~StructFieldValue() noexcept override; void setRepo(const DocumentTypeRepo & repo) { _repo = & repo; } const DocumentTypeRepo * getRepo() const { return _repo; } void setDocumentType(const DocumentType & docType) { _doc_type = & docType; } + const SerializableArray & getFields() const { return _fields; } void lazyDeserialize(const FixedTypeRepo &repo, uint16_t version, SerializableArray::EntryMap && fields, - std::unique_ptr<ByteBuffer> buffer, + ByteBuffer buffer, CompressionConfig::Type comp_type, int32_t uncompressed_length); @@ -74,8 +59,6 @@ public: bool serializeField(int raw_field_id, uint16_t version, FieldValueWriter &writer) const; uint16_t getVersion() const { return _version; } - const Chunks & getChunks() const { return _chunks; } - // raw_ids may contain ids for elements not in the struct's datatype. void getRawFieldIds(std::vector<int> &raw_ids) const; void getRawFieldIds(std::vector<int> &raw_ids, const FieldSet& fieldSet) const; @@ -122,7 +105,6 @@ private: VESPA_DLL_LOCAL vespalib::ConstBufferRef getRawField(uint32_t id) const; VESPA_DLL_LOCAL const StructDataType & getStructType() const; - // Iterator implementation struct FieldIterator; friend struct FieldIterator; diff --git a/document/src/vespa/document/fieldvalue/structuredcache.h b/document/src/vespa/document/fieldvalue/structuredcache.h new file mode 100644 index 00000000000..24d1a3dbd78 --- /dev/null +++ b/document/src/vespa/document/fieldvalue/structuredcache.h @@ -0,0 +1,54 @@ +#pragma once + +#include "fieldvalue.h" +#include <vespa/vespalib/stllike/hash_map.h> + +namespace document { + +class StructuredCache { +public: + using ModificationStatus = fieldvalue::ModificationStatus; + struct ValuePair { + fieldvalue::ModificationStatus status; + FieldValue::UP value; + + ValuePair() : status(ModificationStatus::NOT_MODIFIED), value() {} + + ValuePair(ModificationStatus status_, FieldValue::UP value_) + : status(status_), + value(std::move(value_)) + {} + }; + + using Cache = vespalib::hash_map<Field, ValuePair>; + + void remove(const Field &field) { + ValuePair &entry = _cache[field]; + entry.status = ModificationStatus::REMOVED; + entry.value.reset(); + } + + Cache::iterator find(const Field &field) { + return _cache.find(field); + } + + void set(const Field &field, FieldValue::UP value, ModificationStatus status) { + ValuePair &entry = _cache[field]; + // If the entry has previously been tagged modified, the value we're now reinserting + // is likely based on those changes. We cannot lose that modification status. + entry.status = ((status == ModificationStatus::NOT_MODIFIED) && + (entry.status == ModificationStatus::MODIFIED)) + ? ModificationStatus::MODIFIED + : status; + entry.value = std::move(value); + } + + Cache::iterator begin() { return _cache.begin(); } + + Cache::iterator end() { return _cache.end(); } + +private: + Cache _cache; +}; + +} diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp index 39af319bc8a..2541a253c80 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp @@ -4,6 +4,7 @@ #include "iteratorhandler.h" #include "weightedsetfieldvalue.h" #include "arrayfieldvalue.h" +#include "structuredcache.h" #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> @@ -29,34 +30,6 @@ StructuredFieldValue::Iterator::Iterator(const StructuredFieldValue& owner, cons { } -StructuredFieldValue::StructuredFieldValue(const StructuredFieldValue& other) - : FieldValue(other), - _type(other._type) -{ -} - -StructuredFieldValue::StructuredFieldValue(const DataType &type) - : FieldValue(), - _type(&type) -{ -} - -StructuredFieldValue::~StructuredFieldValue() = default; - - -StructuredFieldValue& -StructuredFieldValue::operator=(const StructuredFieldValue& other) -{ - if (this == &other) { - return *this; - } - - FieldValue::operator=(other); - _type = other._type; - - return *this; -} - void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & value) { @@ -85,53 +58,11 @@ StructuredFieldValue::onGetNestedFieldValue(PathRange nested) const return fv; } -namespace { - struct ValuePair { - fieldvalue::ModificationStatus status; - FieldValue::UP value; - - ValuePair() : status(fieldvalue::ModificationStatus::NOT_MODIFIED), value() {} - ValuePair(fieldvalue::ModificationStatus status_, - FieldValue::UP value_) - : status(status_), - value(std::move(value_)) - {} - }; - - using Cache = vespalib::hash_map<Field, ValuePair>; -} - -class StructuredCache { -public: - void remove(const Field & field) { - ValuePair & entry = _cache[field]; - entry.status = ModificationStatus::REMOVED; - entry.value.reset(); - } - Cache::iterator find(const Field & field) { - return _cache.find(field); - } - void set(const Field & field, FieldValue::UP value, ModificationStatus status) { - ValuePair & entry = _cache[field]; - // If the entry has previously been tagged modified, the value we're now reinserting - // is likely based on those changes. We cannot lose that modification status. - entry.status = ((status == ModificationStatus::NOT_MODIFIED) && - (entry.status == ModificationStatus::MODIFIED)) - ? ModificationStatus::MODIFIED - : status; - entry.value = std::move(value); - } - Cache::iterator begin() { return _cache.begin(); } - Cache::iterator end() { return _cache.end(); } -private: - Cache _cache; -}; - - void StructuredFieldValue::remove(const Field& field) { - if (_cache) { - _cache->remove(field); + StructuredCache * cache = getCache(); + if (cache) { + cache->remove(field); } else { removeFieldValue(field); } @@ -139,8 +70,9 @@ StructuredFieldValue::remove(const Field& field) { void StructuredFieldValue::updateValue(const Field & field, FieldValue::UP value) const { - if (_cache) { - _cache->set(field, std::move(value), ModificationStatus::MODIFIED); + StructuredCache * cache = getCache(); + if (cache) { + cache->set(field, std::move(value), ModificationStatus::MODIFIED); } else { const_cast<StructuredFieldValue&>(*this).setFieldValue(field, std::move(value)); } @@ -148,18 +80,20 @@ StructuredFieldValue::updateValue(const Field & field, FieldValue::UP value) con void StructuredFieldValue::returnValue(const Field & field, FieldValue::UP value) const { - if (_cache) { - _cache->set(field, std::move(value), ModificationStatus::NOT_MODIFIED); + StructuredCache * cache = getCache(); + if (cache) { + cache->set(field, std::move(value), ModificationStatus::NOT_MODIFIED); } } FieldValue::UP StructuredFieldValue::getValue(const Field& field, FieldValue::UP container) const { - if (_cache) { - auto found = _cache->find(field); - if (found == _cache->end()) { + StructuredCache * cache = getCache(); + if (cache) { + auto found = cache->find(field); + if (found == cache->end()) { container = getFieldValue(field); - _cache->set(field, FieldValue::UP(), ModificationStatus::NOT_MODIFIED); + cache->set(field, FieldValue::UP(), ModificationStatus::NOT_MODIFIED); } else { container = std::move(found->second.value); } @@ -238,22 +172,6 @@ StructuredFieldValue::onIterateNested(PathRange nested, IteratorHandler & handle } } -void -StructuredFieldValue::beginTransaction() { - _cache = std::make_unique<StructuredCache>(); -} -void -StructuredFieldValue::commitTransaction() { - for (auto & e : *_cache) { - if (e.second.status == ModificationStatus::REMOVED) { - removeFieldValue(e.first); - } else if (e.second.status == ModificationStatus::MODIFIED) { - setFieldValue(e.first, std::move(e.second.value)); - } - } - _cache.reset(); -} - using ConstCharP = const char *; template void StructuredFieldValue::set(vespalib::stringref field, int32_t value); template void StructuredFieldValue::set(vespalib::stringref field, int64_t value); diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h index b81dc8e3936..54036566a26 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h @@ -33,7 +33,6 @@ class StructuredCache; class StructuredFieldValue : public FieldValue { const DataType *_type; - std::unique_ptr<StructuredCache> _cache; UP onGetNestedFieldValue(PathRange nested) const override; /** @return Retrieve value of given field. Null pointer if not set. @@ -42,12 +41,10 @@ class StructuredFieldValue : public FieldValue VESPA_DLL_LOCAL FieldValue::UP getValue(const Field& field, FieldValue::UP container) const; VESPA_DLL_LOCAL void updateValue(const Field & field, FieldValue::UP value) const; VESPA_DLL_LOCAL void returnValue(const Field & field, FieldValue::UP value) const; + virtual StructuredCache * getCache() const { return nullptr; } protected: - VESPA_DLL_LOCAL StructuredFieldValue(const DataType &type); - StructuredFieldValue(const StructuredFieldValue&); - StructuredFieldValue& operator=(const StructuredFieldValue&); - ~StructuredFieldValue() override; + StructuredFieldValue(const DataType &type) : FieldValue(), _type(&type) {} /** Called from Document when deserializing alters type. */ virtual void setType(const DataType& type) { _type = &type; } @@ -97,6 +94,11 @@ protected: virtual bool hasFieldValue(const Field&) const = 0; virtual void removeFieldValue(const Field&) = 0; virtual FieldValue::UP getFieldValue(const Field&) const = 0; + /** + * Fetches the value of the field and return true if present. + * The document, or the buffer the document was constructed from must live longer than the value. + * This restriction allows lightweight object representation and is significantly faster in many cases. + */ virtual bool getFieldValue(const Field& field, FieldValue& value) const = 0; virtual void setFieldValue(const Field&, FieldValue::UP value) = 0; void setFieldValue(const Field & field, const FieldValue & value); @@ -117,9 +119,6 @@ public: */ virtual const Field& getField(vespalib::stringref name) const = 0; - void beginTransaction(); - void commitTransaction(); - /** * Retrieve value of given field and assign it to given field. * @@ -147,8 +146,12 @@ public: * * @throws vespalib::IllegalArgumentException If value given has wrong type */ - inline void setValue(const Field& field, const FieldValue& value) - { setFieldValue(field, value); } + void setValue(const Field& field, const FieldValue& value) { + setFieldValue(field, value); + } + void setValue(const Field& field, FieldValue::UP value) { + setFieldValue(field, std::move(value)); + } /** Remove the value of given field if it is set. */ //These are affected by the begin/commitTanasaction @@ -157,12 +160,15 @@ public: virtual void clear() = 0; // Utility functions for easy but less efficient access - bool hasValue(vespalib::stringref fieldName) const - { return hasFieldValue(getField(fieldName)); } - void remove(vespalib::stringref fieldName) - { removeFieldValue(getField(fieldName)); } - void setValue(vespalib::stringref fieldName, const FieldValue& value) - { setFieldValue(getField(fieldName), value); } + bool hasValue(vespalib::stringref fieldName) const { + return hasFieldValue(getField(fieldName)); + } + void remove(vespalib::stringref fieldName) { + removeFieldValue(getField(fieldName)); + } + void setValue(vespalib::stringref fieldName, const FieldValue& value) { + setFieldValue(getField(fieldName), value); + } template<typename PrimitiveType> void set(const Field& field, PrimitiveType value); template<typename PrimitiveType> @@ -176,7 +182,7 @@ public: virtual bool empty() const = 0; typedef Iterator const_iterator; - const_iterator begin() const { return const_iterator(*this, NULL); } + const_iterator begin() const { return const_iterator(*this, nullptr); } const_iterator end() const { return const_iterator(); } /** @@ -190,17 +196,5 @@ public: std::unique_ptr<T> getAs(const Field &field) const; }; -class TransactionGuard { -public: - TransactionGuard(StructuredFieldValue & value) - : _value(value) - { - _value.beginTransaction(); - } - ~TransactionGuard() { _value.commitTransaction(); } -private: - StructuredFieldValue & _value; -}; - } // document diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp index 9b318a39f0a..56d7b6ab078 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp @@ -104,6 +104,19 @@ TensorFieldValue::operator=(std::unique_ptr<Tensor> rhs) } +void +TensorFieldValue::make_empty_if_not_existing() +{ + if (!_tensor) { + TensorSpec empty_spec(_dataType.getTensorType().to_spec()); + auto empty_value = Engine::ref().from_spec(empty_spec); + auto tensor_ptr = dynamic_cast<Tensor*>(empty_value.get()); + assert(tensor_ptr != nullptr); + _tensor.reset(tensor_ptr); + empty_value.release(); + } +} + void TensorFieldValue::accept(FieldValueVisitor &visitor) diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h index 80b18be55a0..ea3f8dea9be 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h @@ -28,6 +28,8 @@ public: TensorFieldValue &operator=(const TensorFieldValue &rhs); TensorFieldValue &operator=(std::unique_ptr<vespalib::tensor::Tensor> rhs); + void make_empty_if_not_existing(); + virtual void accept(FieldValueVisitor &visitor) override; virtual void accept(ConstFieldValueVisitor &visitor) const override; virtual const DataType *getDataType() const override; diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h index 15ee0da0c79..52ed8d878a2 100644 --- a/document/src/vespa/document/repo/configbuilder.h +++ b/document/src/vespa/document/repo/configbuilder.h @@ -38,9 +38,9 @@ struct TypeOrId { }; struct Struct : DatatypeConfig { - Struct(const vespalib::string &name) { + explicit Struct(vespalib::string name) { type = Type::STRUCT; - sstruct.name = name; + sstruct.name = std::move(name); } Struct &setCompression(Sstruct::Compression::Type t, int32_t level, int32_t threshold, int32_t min_size) { @@ -63,7 +63,7 @@ struct Struct : DatatypeConfig { }; struct Array : DatatypeConfig { - Array(TypeOrId nested_type) { + explicit Array(TypeOrId nested_type) { addNestedType(nested_type); type = Type::ARRAY; array.element.id = nested_type.id; @@ -71,7 +71,7 @@ struct Array : DatatypeConfig { }; struct Wset : DatatypeConfig { - Wset(TypeOrId nested_type) { + explicit Wset(TypeOrId nested_type) { addNestedType(nested_type); type = Type::WSET; wset.key.id = nested_type.id; @@ -94,7 +94,7 @@ struct Map : DatatypeConfig { }; struct AnnotationRef : DatatypeConfig { - AnnotationRef(int32_t annotation_type_id) { + explicit AnnotationRef(int32_t annotation_type_id) { type = Type::ANNOTATIONREF; annotationref.annotation.id = annotation_type_id; } @@ -103,7 +103,7 @@ struct AnnotationRef : DatatypeConfig { struct DocTypeRep { DocumenttypesConfig::Documenttype &doc_type; - DocTypeRep(DocumenttypesConfig::Documenttype &type) + explicit DocTypeRep(DocumenttypesConfig::Documenttype &type) : doc_type(type) {} DocTypeRep &inherit(int32_t id) { doc_type.inherits.resize(doc_type.inherits.size() + 1); @@ -127,6 +127,12 @@ struct DocTypeRep { doc_type.referencetype.back().targetTypeId = target_type_id; return *this; } + + DocTypeRep& imported_field(vespalib::string field_name) { + doc_type.importedfield.resize(doc_type.importedfield.size() + 1); + doc_type.importedfield.back().name = std::move(field_name); + return *this; + } }; class DocumenttypesConfigBuilderHelper { diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index da59f527115..58dbd16beb6 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -441,10 +441,10 @@ void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, Do for (const auto & entry : fsv) { const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second); DocumentType::FieldSet::Fields fields; - for (size_t j(0); j < fs.fields.size(); j++) { - fields.insert(fs.fields[j]); + for (const auto& f : fs.fields) { + fields.insert(f); } - doc_type.addFieldSet(entry.first, fields); + doc_type.addFieldSet(entry.first, std::move(fields)); } } @@ -457,6 +457,14 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference } } +void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldVector& imported_fields, + DocumentType& doc_type) +{ + for (const auto& imported : imported_fields) { + doc_type.add_imported_field_name(imported.name); + } +} + void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) { DataTypeRepo *data_types = type_map[doc_type.id]; inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations); @@ -467,6 +475,7 @@ void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, Do setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo); inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type); addFieldSet(doc_type.fieldsets, *data_types->doc_type); + add_imported_fields(doc_type.importedfield, *data_types->doc_type); } void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) { diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index eaa795549bf..ef2ee26bdbd 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -191,35 +191,64 @@ GlobOperator::traceImpl(const Value& a, const Value& b, std::ostream& ost) const return match(left->getValue(), regex); } +namespace { + +// Returns the number of consecutive wildcard ('*') characters found from +// _and including_ the character at `i`, i.e. the wildcard run length. +size_t wildcard_run_length(size_t i, vespalib::stringref str) { + size_t n = 0; + for (; (i < str.size()) && (str[i] == '*'); ++n, ++i) {} + return n; +} + +} + vespalib::string -GlobOperator::convertToRegex(vespalib::stringref globpattern) const +GlobOperator::convertToRegex(vespalib::stringref globpattern) { + if (globpattern.empty()) { + return "^$"; // Empty glob can only match the empty string. + } vespalib::asciistream ost; - ost << '^'; - for(uint32_t i=0, n=globpattern.size(); i<n; ++i) { + size_t i = 0; + if (globpattern[0] != '*') { + ost << '^'; + } else { + i += wildcard_run_length(0, globpattern); // Skip entire prefix wildcard run. + } + const size_t n = globpattern.size(); + for (; i < n; ++i) { switch(globpattern[i]) { - case '*': ost << ".*"; - break; - case '?': ost << "."; - break; - case '^': - case '$': - case '|': - case '{': - case '}': - case '(': - case ')': - case '[': - case ']': - case '\\': - case '+': - case '.': ost << '\\' << globpattern[i]; - break; - // Are there other regex special chars we need to escape? - default: ost << globpattern[i]; + case '*': + i += wildcard_run_length(i, globpattern) - 1; // -1 since we always inc by 1 anyway. + if (i != (n - 1)) { // Don't emit trailing wildcard. + ost << ".*"; + } + break; + case '?': + ost << '.'; + break; + case '^': + case '$': + case '|': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case '\\': + case '+': + case '.': + ost << '\\' << globpattern[i]; + break; + // Are there other regex special chars we need to escape? + default: ost << globpattern[i]; } } - ost << '$'; + if (globpattern[n - 1] != '*') { + ost << '$'; + } return ost.str(); } diff --git a/document/src/vespa/document/select/operator.h b/document/src/vespa/document/select/operator.h index 7383b6cf545..ad73d284c63 100644 --- a/document/src/vespa/document/select/operator.h +++ b/document/src/vespa/document/select/operator.h @@ -88,7 +88,27 @@ public: // Delegates to Value::globCompare ResultList compare(const Value& a, const Value& b) const override; ResultList trace(const Value&, const Value&, std::ostream& trace) const override; - vespalib::string convertToRegex(vespalib::stringref globpattern) const; + /** + * Converts a standard glob expression into a regular expression string, + * supporting the following glob semantics: + * '*' matches 0-n arbitrary characters + * '?' matches exactly 1 arbitrary character + * This code simplifies the resulting regex as much as possible to help + * minimize the number of possible catastrophic backtracking cases that + * can be triggered by wildcard regexes. + * + * The following simplifications are currently performed: + * - '' -> /^$/ (empty string match) + * '*' -> // (any string match) + * - '*foo*' -> /foo/ (substring match) + * - '*foo' -> /foo$/ (suffix match) + * - 'foo*' -> /^foo/ (prefix match) + * - collapsing runs of consecutive '*' wildcards into a single + * wildcard. *** is identical to ** which is identical to * etc, + * as all these match 0-n characters each. This also works with + * simplification, i.e. '***foo***' -> /foo/ and '***' -> // + */ + static vespalib::string convertToRegex(vespalib::stringref globpattern); static bool containsVariables(vespalib::stringref expression); static const GlobOperator GLOB; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 5c7820b76d7..95cf2f4e7e5 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -340,10 +340,34 @@ FieldValueNode::initFieldPath(const DocumentType& type) const { } } +namespace { + +bool looks_like_complex_field_path(const vespalib::string& expr) { + for (const char c : expr) { + switch (c) { + case '.': + case '[': + case '{': + return true; + default: continue; + } + } + return false; +} + +bool is_simple_imported_field(const vespalib::string& expr, const DocumentType& doc_type) { + if (looks_like_complex_field_path(expr)) { + return false; + } + return (doc_type.has_imported_field_name(expr)); +} + +} + std::unique_ptr<Value> FieldValueNode::getValue(const Context& context) const { - if (context._doc == NULL) { + if (context._doc == nullptr) { return std::make_unique<InvalidValue>(); } @@ -352,7 +376,17 @@ FieldValueNode::getValue(const Context& context) const if (!documentTypeEqualsName(doc.getType(), _doctype)) { return std::make_unique<InvalidValue>(); } - try{ + // Imported fields can only be meaningfully evaluated inside Proton, so we + // explicitly treat them as if they are valid fields with missing values. This + // will be treated the same as if it's a normal field by the selection operators. + // This avoids any awkward interaction with Invalid values or having to + // augment the FieldPath code with knowledge of imported fields. + // When a selection is running inside Proton, it will patch FieldValueNodes for + // imported fields, which removes this check entirely. + if (is_simple_imported_field(_fieldExpression, doc.getType())) { + return std::make_unique<NullValue>(); + } + try { initFieldPath(doc.getType()); IteratorHandler handler; @@ -363,7 +397,7 @@ FieldValueNode::getValue(const Context& context) const } else { const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - if (values.size() == 0) { + if (values.empty()) { return std::make_unique<NullValue>(); } else { return std::make_unique<ArrayValue>(handler.getValues()); @@ -399,7 +433,7 @@ FieldValueNode::print(std::ostream& out, bool verbose, std::unique_ptr<Value> FieldValueNode::traceValue(const Context &context, std::ostream& out) const { - if (context._doc == NULL) { + if (context._doc == nullptr) { return defaultTrace(getValue(context), out); } const Document &doc(*context._doc); @@ -408,7 +442,12 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const << _doctype << " document, thus resolving invalid.\n"; return std::make_unique<InvalidValue>(); } - try{ + if (is_simple_imported_field(_fieldExpression, doc.getType())) { + out << "Field '" << _fieldExpression << "' refers to an imported field; " + << "returning NullValue to treat this as an unset field value.\n"; + return std::make_unique<NullValue>(); + } + try { initFieldPath(doc.getType()); IteratorHandler handler; diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h index cbcc2c97017..0e0bfabdde5 100644 --- a/document/src/vespa/document/serialization/util.h +++ b/document/src/vespa/document/serialization/util.h @@ -2,6 +2,8 @@ #pragma once +#include <cstdint> + namespace document { // Sets the value of a variable for the duration of this object's lifetime. @@ -97,32 +99,5 @@ void putInt2_4_8Bytes(Output &out, uint64_t val) { } } -inline uint32_t sizeOfInt1_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt1_2_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else if (val < 0x4000) { - return 2; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt2_4_8Bytes(uint64_t val) { - if (val < 0x8000) { - return 2; - } else if (val < 0x40000000) { - return 4; - } else { - return 8; - } -} } // namespace document diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 4fe7cfc6d29..94644438f5c 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -2,7 +2,6 @@ #include "vespadocumentdeserializer.h" #include "annotationdeserializer.h" -#include <vespa/document/annotation/spantree.h> #include <vespa/document/fieldvalue/annotationreferencefieldvalue.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/boolfieldvalue.h> @@ -19,7 +18,6 @@ #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -30,6 +28,8 @@ #include <vespa/document/base/exceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bytebuffer.h> +#include <vespa/document/base/idstringexception.h> + #include <vespa/log/log.h> LOG_SETUP(".vespadocumentdeserializer"); @@ -74,15 +74,15 @@ void VespaDocumentDeserializer::readDocument(Document &value) { if (type) { Document::verifyIdAndType(value.getId(), type); value.setType(*type); - value.clear(); value.setLastModified(0); + } else { + value.getFields().reset(); } value.setRepo(_repo.getDocumentTypeRepo()); FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType()); VarScope<FixedTypeRepo> repo_scope(_repo, repo); uint32_t chunkCount = getChunkCount(content_code); - value.getFields().reset(); for (uint32_t i = 0; i < chunkCount; ++i) { readStructNoReset(value.getFields()); } @@ -306,13 +306,29 @@ void VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) { } if (data_size > 0) { - ByteBuffer::UP buffer(_stream.isLongLivedBuffer() - ? new ByteBuffer(_stream.peek(), data_size) + ByteBuffer buffer(_stream.isLongLivedBuffer() + ? ByteBuffer(_stream.peek(), data_size) : ByteBuffer::copyBuffer(_stream.peek(), data_size)); - LOG(spam, "Lazy deserializing into %s with _version %u", - value.getDataType()->getName().c_str(), _version); - value.lazyDeserialize(_repo, _version, std::move(field_info), - std::move(buffer), compression_type, uncompressed_size); + if (value.getFields().empty()) { + LOG(spam, "Lazy deserializing into %s with _version %u", + value.getDataType()->getName().c_str(), _version); + value.lazyDeserialize(_repo, _version, std::move(field_info), + std::move(buffer), compression_type, uncompressed_size); + } else { + LOG(debug, "Legacy dual header/body format. -> Merging."); + StructFieldValue tmp(*value.getDataType()); + tmp.lazyDeserialize(_repo, _version, std::move(field_info), + std::move(buffer), compression_type, uncompressed_size); + for (const auto & entry : tmp) { + try { + FieldValue::UP decoded = tmp.getValue(entry); + value.setValue(entry, std::move(decoded)); + } catch (const vespalib::Exception & e) { + LOG(warning, "Failed decoding field '%s' in legacy bodyfield -> Skipping it: %s", + entry.getName().data(), e.what()); + } + } + } _stream.adjustReadPos(data_size); } } @@ -352,9 +368,7 @@ VespaDocumentDeserializer::read(TensorFieldValue &value) nbostream wrapStream(_stream.peek(), length); tensor = vespalib::tensor::TypedBinaryFormat::deserialize(wrapStream); if (wrapStream.size() != 0) { - throw DeserializeException("Leftover bytes deserializing " - "tensor field value.", - VESPA_STRLOC); + throw DeserializeException("Leftover bytes deserializing tensor field value.", VESPA_STRLOC); } } value.assignDeserialized(std::move(tensor)); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 570a51907da..eadbd4b5a8a 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -86,89 +86,45 @@ void VespaDocumentSerializer::write(const DocumentType &value) { << static_cast<uint16_t>(0); // version } +namespace { + uint8_t -VespaDocumentSerializer::getContentCode(bool hasHeader, bool hasBody) const +getContentCode(bool hasContent) { - uint8_t content = 0x01; // Document type is always present. - if (hasHeader) { - content |= 0x02; // Header is present. - } - if (hasBody) { - content |= 0x04; // Body is present. - } - return content; + return 0x01u | // Document type is always present + (hasContent ? 0x02u : 0x00u); // Payload ? } -static inline size_t wantChunks(bool hasHeader, bool hasBody) { - size_t res = 0; - if (hasHeader) ++res; - if (hasBody) ++res; - return res; } void -VespaDocumentSerializer::write(const Document &value, DocSerializationMode mode) { +VespaDocumentSerializer::write(const Document &value) { nbostream doc_stream; VespaDocumentSerializer doc_serializer(doc_stream); doc_serializer.write(value.getId()); - bool hasHeader = false; - bool hasBody = false; - - const StructFieldValue::Chunks & chunks = value.getFields().getChunks(); - - for (const Field & field : value.getFields()) { - if (field.isHeaderField()) { - hasHeader = true; - } else { - hasBody = true; - } - if (hasHeader && hasBody) { - break; - } - } - if (mode != COMPLETE) { - hasBody = false; - } - doc_stream << getContentCode(hasHeader, hasBody); + bool hasContent = ! value.getFields().empty(); + doc_stream << getContentCode(hasContent); doc_serializer.write(value.getType()); - if (chunks.size() == wantChunks(hasHeader, hasBody) && - !structNeedsReserialization(value.getFields())) - { - // here we assume the receiver can handle whatever serialization the - // chunks contain, so we just send them as-is, even if some fields - // may have moved from header to body or vice versa. - if (hasHeader || hasBody) { - assert( ! chunks.empty()); - doc_serializer.writeUnchanged(chunks[0]); - } - if (hasHeader && hasBody) { - assert(chunks.size() == 2); - doc_serializer.writeUnchanged(chunks[1]); - } - } else { - if (hasHeader) { - doc_serializer.write(value.getFields(), HeaderFields()); - } - if (hasBody) { - doc_serializer.write(value.getFields(), BodyFields()); + if ( hasContent ) { + if (!structNeedsReserialization(value.getFields())) { + doc_serializer.writeUnchanged(value.getFields().getFields()); + } else { + doc_serializer.write(value.getFields(), AllFields()); } } const uint16_t version = serialize_version; - _stream << version - << static_cast<uint32_t>(doc_stream.size()); + _stream << version << static_cast<uint32_t>(doc_stream.size()); _stream.write(doc_stream.peek(), doc_stream.size()); } void VespaDocumentSerializer::visit(const StructFieldValue &value) { - const StructFieldValue::Chunks & chunks = value.getChunks(); - if (!structNeedsReserialization(value) && ! chunks.empty()) { - assert(chunks.size() == 1); - writeUnchanged(chunks[0]); + if (!structNeedsReserialization(value)) { + writeUnchanged(value.getFields()); } else { write(value, AllFields()); } @@ -247,7 +203,7 @@ VespaDocumentSerializer::write(const ShortFieldValue &value) { void VespaDocumentSerializer::write(const StringFieldValue &value) { - uint8_t coding = (value.hasSpanTrees() << 6); + uint8_t coding = (value.hasSpanTrees() << 6u); _stream << coding; putInt1_4Bytes(_stream, value.getValueRef().size() + 1); _stream.write(value.getValueRef().data(), value.getValueRef().size()); @@ -293,10 +249,10 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { using vespalib::compression::compress; - vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); + vespalib::ConstBufferRef buf(stream.data(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, - vespalib::ConstBufferRef(stream.c_str(), stream.size()), + vespalib::ConstBufferRef(stream.data(), stream.size()), compressed_data, false); if (compressedType != config.type || ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) @@ -341,17 +297,8 @@ VespaDocumentSerializer::structNeedsReserialization(const StructFieldValue &valu return false; } - const StructFieldValue::Chunks & chunks = value.getChunks(); - - for (uint32_t i = 0; i < chunks.size(); ++i) { - if (chunks[i].getCompression() != value.getCompressionConfig().type && - chunks[i].getCompression() != CompressionConfig::UNCOMPRESSABLE) - { - return true; - } - } - - return false; + return (value.getFields().getCompression() != value.getCompressionConfig().type && + value.getFields().getCompression() != CompressionConfig::UNCOMPRESSABLE); } void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { @@ -378,8 +325,7 @@ void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { } } -void VespaDocumentSerializer::write(const StructFieldValue &value, - const FieldSet& fieldSet) +void VespaDocumentSerializer::write(const StructFieldValue &value, const FieldSet& fieldSet) { nbostream value_stream; vector<pair<uint32_t, uint32_t> > field_info; diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index d83532771e8..f664a32d893 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -17,13 +17,11 @@ class SerializableArray; class ValueUpdate; class FieldPathUpdate; -enum DocSerializationMode { COMPLETE, WITHOUT_BODY }; - class VespaDocumentSerializer : private ConstFieldValueVisitor, private UpdateVisitor, public FieldValueWriter { public: - VespaDocumentSerializer(vespalib::nbostream &stream); + explicit VespaDocumentSerializer(vespalib::nbostream &stream); static bool structNeedsReserialization(const StructFieldValue &value); @@ -34,7 +32,7 @@ public: void write(const DocumentId &value); void write(const DocumentType &value); - void write(const Document &value, DocSerializationMode mode); + void write(const Document &value); void write(const AnnotationReferenceFieldValue &value); void write(const ArrayFieldValue &value); void write(const MapFieldValue &map); @@ -60,8 +58,6 @@ public: private: static constexpr int serialize_version = 8; void writeUnchanged(const SerializableArray &val); - uint8_t getContentCode(bool hasHeader, bool hasBody) const; - void write(const FieldPathUpdate &value); void write(const RemoveValueUpdate &value); @@ -96,7 +92,7 @@ private: void visit(const ArrayFieldValue &value) override { write(value); } void visit(const BoolFieldValue &value) override { write(value); } void visit(const ByteFieldValue &value) override { write(value); } - void visit(const Document &value) override { write(value, COMPLETE); } + void visit(const Document &value) override { write(value); } void visit(const DoubleFieldValue &value) override { write(value); } void visit(const FloatFieldValue &value) override { write(value); } void visit(const IntFieldValue &value) override { write(value); } diff --git a/document/src/vespa/document/update/assignfieldpathupdate.h b/document/src/vespa/document/update/assignfieldpathupdate.h index a1349bab96a..718d001e060 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.h +++ b/document/src/vespa/document/update/assignfieldpathupdate.h @@ -11,9 +11,9 @@ class AssignFieldPathUpdate : public FieldPathUpdate public: enum SerializationFlag { - ARITHMETIC_EXPRESSION = 1, - REMOVE_IF_ZERO = 2, - CREATE_MISSING_PATH = 4 + ARITHMETIC_EXPRESSION = 1u, + REMOVE_IF_ZERO = 2u, + CREATE_MISSING_PATH = 4u }; /** For deserialization */ diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index b43b7a59c5b..f289e6a8f27 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -7,7 +7,6 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bufferexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -231,20 +230,18 @@ DocumentUpdate::serializeFlags(int size_) const } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream && stream) { - vespalib::nbostream is(buffer.getBufferAtPos(), buffer.getRemaining()); auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, is); - buffer.setPos(buffer.getPos() + is.rp()); + update->initHEAD(repo, std::move(stream)); return update; } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream stream) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, std::move(stream)); + update->initHEAD(repo, stream); return update; } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index a672d028f35..fb3d6c0f7a3 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -32,6 +32,7 @@ namespace document { +class ByteBuffer; class Document; class VespaDocumentSerializer; /** @@ -50,8 +51,8 @@ public: /** * Create new style document update, possibly with field path updates. */ - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream stream); - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, ByteBuffer & buffer); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); DocumentUpdate(); /** @@ -124,8 +125,8 @@ private: bool _needHardReserialize; int deserializeFlags(int sizeAndFlags); - void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream); void lazyDeserialize(const DocumentTypeRepo & repo, vespalib::nbostream & stream); void ensureDeserialized() const; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 3e19420d0d0..0dbf8e5cc9e 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -7,7 +7,6 @@ namespace document { -class ByteBuffer; class DocumentTypeRepo; class Field; class FieldValue; @@ -35,7 +34,7 @@ public: using SP = std::shared_ptr<FieldPathUpdate>; using CP = vespalib::CloneablePtr<FieldPathUpdate>; - ~FieldPathUpdate(); + ~FieldPathUpdate() override; enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), diff --git a/document/src/vespa/document/update/tensor_add_update.cpp b/document/src/vespa/document/update/tensor_add_update.cpp index c35a8058133..2e5fa194c20 100644 --- a/document/src/vespa/document/update/tensor_add_update.cpp +++ b/document/src/vespa/document/update/tensor_add_update.cpp @@ -93,6 +93,7 @@ TensorAddUpdate::applyTo(FieldValue& value) const { if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); + tensorFieldValue.make_empty_if_not_existing(); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); auto newTensor = applyTo(*oldTensor); if (newTensor) { diff --git a/document/src/vespa/document/update/tensor_modify_update.cpp b/document/src/vespa/document/update/tensor_modify_update.cpp index 0d821de8922..dfc7479e5cd 100644 --- a/document/src/vespa/document/update/tensor_modify_update.cpp +++ b/document/src/vespa/document/update/tensor_modify_update.cpp @@ -174,9 +174,11 @@ TensorModifyUpdate::applyTo(FieldValue& value) const if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); - auto newTensor = applyTo(*oldTensor); - if (newTensor) { - tensorFieldValue = std::move(newTensor); + if (oldTensor) { + auto newTensor = applyTo(*oldTensor); + if (newTensor) { + tensorFieldValue = std::move(newTensor); + } } } else { vespalib::string err = make_string("Unable to perform a tensor modify update on a '%s' field value", diff --git a/document/src/vespa/document/update/tensor_remove_update.cpp b/document/src/vespa/document/update/tensor_remove_update.cpp index f3bf7da7a0b..91b4c0a6ca3 100644 --- a/document/src/vespa/document/update/tensor_remove_update.cpp +++ b/document/src/vespa/document/update/tensor_remove_update.cpp @@ -120,9 +120,11 @@ TensorRemoveUpdate::applyTo(FieldValue &value) const if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); - auto newTensor = applyTo(*oldTensor); - if (newTensor) { - tensorFieldValue = std::move(newTensor); + if (oldTensor) { + auto newTensor = applyTo(*oldTensor); + if (newTensor) { + tensorFieldValue = std::move(newTensor); + } } } else { vespalib::string err = make_string("Unable to perform a tensor remove update on a '%s' field value", diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 8cb148abe25..48ca7bd36d7 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp printable.cpp - serializable.cpp + serializableexceptions.cpp stringutil.cpp DEPENDS AFTER diff --git a/document/src/vespa/document/util/bufferexceptions.h b/document/src/vespa/document/util/bufferexceptions.h index 8a3215f6c79..aee7f3ae568 100644 --- a/document/src/vespa/document/util/bufferexceptions.h +++ b/document/src/vespa/document/util/bufferexceptions.h @@ -15,13 +15,5 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(BufferOutOfBoundsException) }; -class InputOutOfRangeException : public vespalib::IoException { -public: - InputOutOfRangeException(const vespalib::string& msg, - const vespalib::string& location = ""); - - VESPA_DEFINE_EXCEPTION_SPINE(InputOutOfRangeException) -}; - } diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index ccbc2bc7790..644edb0664d 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -12,732 +12,159 @@ #include <sstream> #include <arpa/inet.h> -#define LOG_DEBUG1(a) -// Enable this macros instead to see what bytebuffer calls come -//#define LOG_DEBUG1(a) std::cerr << "ByteBuffer(" << ((void*) this) << " " << a << ")\n"; - -#define LOG_DEBUG2(a,b) LOG_DEBUG1(vespalib::make_string(a,b)); -#define LOG_DEBUG3(a,b,c) LOG_DEBUG1(vespalib::make_string(a,b,c)); -#define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d)); - using vespalib::alloc::Alloc; +using vespalib::make_string; namespace document { -VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(InputOutOfRangeException); +namespace { -vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { - vespalib::asciistream ost; - ost << pos << " > " << len; - return ost.str(); -} - -BufferOutOfBoundsException::BufferOutOfBoundsException( - size_t pos, size_t len, const vespalib::string& location) - : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) -{ -} - -InputOutOfRangeException::InputOutOfRangeException( - const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::INTERNAL_FAILURE, location, 1) -{ -} +static void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline, noreturn)); -ByteBuffer::ByteBuffer() : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() +void throwOutOfBounds(size_t want, size_t has) { - set(NULL, 0); - LOG_DEBUG1("Created empty bytebuffer"); + throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); } -ByteBuffer::ByteBuffer(size_t len) : - ByteBuffer(Alloc::alloc(len), len) -{ } -ByteBuffer::ByteBuffer(const char* buffer, size_t len) : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(buffer, len); -} +#if defined(__i386__) || defined(__x86_64__) -ByteBuffer::ByteBuffer(Alloc buffer, size_t len) : - _buffer(static_cast<char *>(buffer.get())), - _len(len), - _pos(0), - _limit(len), - _bufHolder(NULL), - _ownedBuffer(std::move(buffer)) -{ -} +template<typename T> +void +ByteBuffer::getDoubleLongNetwork(T &val) { + //TODO: Change this if we move to big-endian hardware + if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { + throwOutOfBounds(sizeof(T), getRemaining()); + } -ByteBuffer::ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit) : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(buf, pos, len, limit); - LOG_DEBUG3("Created copy of byte buffer of length %" PRIu64 " with " - "limit %" PRIu64 ".", len, limit); + auto * data = reinterpret_cast<unsigned char*>(&val); + for (int i=sizeof(T)-1; i>=0; --i) { + getByte(data[i]); + } } -ByteBuffer::ByteBuffer(const ByteBuffer& bb) : - _buffer(0), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - LOG_DEBUG1("Created empty byte buffer to assign to."); - *this = bb; -} +#else +#error "getDoubleLongNetwork is undefined for this arcitecture" +#endif -ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org) -{ - if (this != & org) { - cleanUp(); - if (org._len > 0 && org._buffer) { - Alloc::alloc(org._len + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer,org._buffer,org._len); - _buffer[org._len] = 0; - } - _len = org._len; - _pos = org._pos; - _limit = org._limit; - LOG_DEBUG4("Assignment created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); - } - return *this; -} +VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); -void -ByteBuffer::set(BufferHolder* buf, size_t pos, size_t len, size_t limit) -{ - cleanUp(); - _bufHolder = buf; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=pos; - _len=len; - _limit=limit; - LOG_DEBUG4("set() created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); +vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { + vespalib::asciistream ost; + ost << pos << " > " << len; + return ost.str(); } -ByteBuffer::~ByteBuffer() +BufferOutOfBoundsException::BufferOutOfBoundsException(size_t pos, size_t len, const vespalib::string& location) + : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) { - if (_bufHolder) { - _bufHolder->subRef(); - } } -std::unique_ptr<ByteBuffer> -ByteBuffer::sliceCopy() const +ByteBuffer::ByteBuffer(Alloc buffer, uint32_t len) + : _buffer(static_cast<const char *>(buffer.get())), + _len(len), + _pos(0), + _ownedBuffer(std::make_unique<Alloc>(std::move(buffer))) { - ByteBuffer* buf = new ByteBuffer; - buf->sliceFrom(*this, _pos, _limit); - - LOG_DEBUG3("Created slice at pos %" PRIu64 " with limit %" PRIu64 ".", - _pos, _limit); - return std::unique_ptr<ByteBuffer>(buf); } -void ByteBuffer::throwOutOfBounds(size_t want, size_t has) +ByteBuffer::ByteBuffer(std::unique_ptr<Alloc> buffer, uint32_t len) + : _buffer(static_cast<const char *>(buffer->get())), + _len(len), + _pos(0), + _ownedBuffer(std::move(buffer)) { - LOG_DEBUG1("Throwing out of bounds exception"); - throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); } -void -ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (BufferOutOfBoundsException) +ByteBuffer::ByteBuffer(const ByteBuffer& rhs) + : _buffer(nullptr), + _len(rhs._len), + _pos(rhs._pos), + _ownedBuffer() { - LOG_DEBUG3("Created slice from buffer from %" PRIu64 " to %" PRIu64 ".", - from, to); - if (from > buf._len) { - throwOutOfBounds(from, buf._len); - } else if (to > buf._len) { - throwOutOfBounds(to, buf._len); - } else if (to < from) { - throwOutOfBounds(to, from); - } else { - - if (!buf._buffer) { - clear(); - return; - } - - // Slicing from someone that doesn't own their buffer, must make own copy. - if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) { - cleanUp(); - Alloc::alloc(to-from + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer, buf._buffer + from, to-from); - _buffer[to-from] = 0; - _pos = 0; - _len = _limit = to-from; - return; - } - - // Slicing from someone that owns, but hasn't made a reference counter yet. - if (!buf._bufHolder) { - buf._bufHolder=new BufferHolder(std::move(const_cast<Alloc &>(buf._ownedBuffer))); - } - - // Slicing from refcounter. - cleanUp(); - - _bufHolder = buf._bufHolder; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=from; - _len=to; - _limit=to; + if (rhs._len > 0 && rhs._buffer) { + auto buf = Alloc::alloc(rhs._len); + memcpy(buf.get(), rhs._buffer, rhs._len); + _buffer = static_cast<const char *>(buf.get()); + _ownedBuffer = std::make_unique<Alloc>(std::move(buf)); } } -ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len) +ByteBuffer +ByteBuffer::copyBuffer(const char* buffer, uint32_t len) { if (buffer && len) { - Alloc newBuf = Alloc::alloc(len + 1); + Alloc newBuf = Alloc::alloc(len); memcpy(newBuf.get(), buffer, len); - static_cast<char *>(newBuf.get())[len] = 0; - return new ByteBuffer(std::move(newBuf), len); - } else { - return NULL; - } -} - -void -ByteBuffer::setPos(size_t pos) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting pos to be %" PRIu64 ", limit is %" PRIu64 ".", - pos, _limit); - if (pos>_limit) { - throwOutOfBounds(pos, _limit); - } else { - _pos=pos; - } -} - -void -ByteBuffer::setLimit(size_t limit) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting limit to %" PRIu64 ", (size is %" PRIu64 ").", limit, _len); - if (limit>_len) { - throwOutOfBounds(limit, _len); + return ByteBuffer(std::make_unique<Alloc>(std::move(newBuf)), len); } else { - _limit=limit; + return ByteBuffer(); } } - -ByteBuffer::BufferHolder::BufferHolder(Alloc buffer) - : _buffer(std::move(buffer)) -{ -} - -ByteBuffer::BufferHolder::~BufferHolder() = default; -void ByteBuffer::dump() const +void ByteBuffer::incPos(uint32_t pos) { - fprintf(stderr, "ByteBuffer: Length %lu, Pos %lu, Limit %lu\n", - _len, _pos, _limit); - for (size_t i=0; i<_len; i++) { - if (_buffer[i]>32 && _buffer[i]<126) { - fprintf(stderr, "%c", _buffer[i]); - } else { - fprintf(stderr, "[%d]",_buffer[i]); - } - } -} - -void ByteBuffer::incPos(size_t pos) -{ - LOG_DEBUG2("incPos(%" PRIu64 ")", pos); - if (_pos + pos > _limit) { - throwOutOfBounds(_pos + pos, _limit); + if (_pos + pos > _len) { + throwOutOfBounds(_pos + pos, _len); } else { _pos+=pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif } } void ByteBuffer::getNumeric(uint8_t & v) { - LOG_DEBUG2("getNumeric8(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(uint8_t *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(uint8_t v) { - LOG_DEBUG2("putNumeric8(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - *(uint8_t *) getBufferAtPos() = v; + v = *reinterpret_cast<const uint8_t *>(getBufferAtPos()); incPosNoCheck(sizeof(v)); } } -size_t ByteBuffer::forceValgrindStart2Pos() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(_buffer), *e(c + _pos); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - -size_t ByteBuffer::forceValgrindPos2Lim() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(getBufferAtPos()), *e(c + getRemaining()); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - - void ByteBuffer::getNumericNetwork(int16_t & v) { - LOG_DEBUG2("getNumericNetwork16(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint16_t val = *(uint16_t *) (void *) getBufferAtPos(); + uint16_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohs(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int16_t & v) { - LOG_DEBUG2("getNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int16_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(int16_t v) { - LOG_DEBUG2("putNumericNetwork16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint16_t val = htons(v); - *(uint16_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int16_t v) { - LOG_DEBUG2("putNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int16_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - void ByteBuffer::getNumericNetwork(int32_t & v) { - LOG_DEBUG2("getNumericNetwork32(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint32_t val = *(uint32_t *) (void *) getBufferAtPos(); + uint32_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohl(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int32_t & v) { - LOG_DEBUG2("getNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int32_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - - -void ByteBuffer::putNumericNetwork(int32_t v) { - LOG_DEBUG2("putNumericNetwork32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint32_t val = htonl(v); - *(uint32_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int32_t v) { - LOG_DEBUG2("putNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int32_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::getNumericNetwork(float & v) { - LOG_DEBUG2("getNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(uint32_t) == 4 - // and endianness same for float and ints - int32_t val; - getIntNetwork(val); - memcpy(&v, &val, sizeof(v)); -} - -void ByteBuffer::getNumeric(float & v) { - LOG_DEBUG2("getNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(float *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(float v) { - LOG_DEBUG2("putNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(int32_t) == 4 - // and endianness same for float and ints - int32_t val; - memcpy(&val, &v, sizeof(val)); - putIntNetwork(val); -} - -void ByteBuffer::putNumeric(float v) { - LOG_DEBUG2("putNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(float *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} void ByteBuffer::getNumeric(int64_t& v) { - LOG_DEBUG2("getNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int64_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(int64_t v) { - LOG_DEBUG2("putNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int64_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::getNumeric(double& v) { - LOG_DEBUG2("getNumericDouble(%f)", v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - v = *(double *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(double v) { - LOG_DEBUG2("putNumericDouble(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(double *) (void *) getBufferAtPos() = v; + memcpy(&v, getBufferAtPos(), sizeof(v)); incPosNoCheck(sizeof(v)); } } void ByteBuffer::getNumericNetwork(double & v) { - LOG_DEBUG2("getNumericNetworkDouble(%f)", v); getDoubleLongNetwork(v); } -void ByteBuffer::putNumericNetwork(int64_t v) { - LOG_DEBUG2("putNumericNetwork64(%" PRId64 ")", v); - putDoubleLongNetwork(v); -} -void ByteBuffer::putNumericNetwork(double v) { - LOG_DEBUG2("putNumericNetworkDouble(%f)", v); - putDoubleLongNetwork(v); -} + void ByteBuffer::getNumericNetwork(int64_t & v) { - LOG_DEBUG2("getNumericNetwork64(%" PRId64 ")", v); getDoubleLongNetwork(v); } -void ByteBuffer::putInt2_4_8Bytes(int64_t number, size_t len) { - LOG_DEBUG3("putInt2_4_8(%" PRId64 ", %" PRIu64 ")", number, len); - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (len == 0) { - if (number < 0x8000ll) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (number < 0x40000000ll) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } - } else if (len == 2) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (len == 4) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else if (len == 8) { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } else { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number using %d bytes.", (int)len), VESPA_STRLOC); - } -} - -void ByteBuffer::getInt2_4_8Bytes(int64_t & v) { - LOG_DEBUG2("getInt2_4_8(%" PRId64 ")", v); - if (getRemaining() >= 2) { - uint8_t flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 8 bytes - int64_t tmp; - getLongNetwork(tmp); - v = tmp & 0x3FFFFFFFFFFFFFFFll; - } else { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = (int64_t) (tmp & 0x3FFFFFFF); - } - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int64_t) tmp; - } - } else { - throwOutOfBounds(getRemaining(), 2); - } -} - -size_t ByteBuffer::getSerializedSize2_4_8Bytes(int64_t number) { - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (number < 0x8000ll) { - return 2; - } else if (number < 0x40000000ll) { - return 4; - } else { - return 8; - } -} - -void ByteBuffer::putInt1_2_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_2_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else if (number < 0x4000) { - putShortNetwork((int16_t) (((int16_t)number) | ((int16_t) 0x8000))); - } else { - putIntNetwork(number | 0xC0000000); - } -} - -void ByteBuffer::getInt1_2_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_2_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x3FFFFFFF; - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int32_t) (tmp & ((int16_t) 0x3FFF)); - } - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} - -size_t ByteBuffer::getSerializedSize1_2_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else if (number < 0x4000) { - return 2; - } else { - return 4; - } -} -void ByteBuffer::putInt1_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else { - putIntNetwork(number | 0x80000000); - } -} -void ByteBuffer::getInt1_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x7FFFFFFF; - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} -size_t ByteBuffer::getSerializedSize1_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else { - return 4; - } -} -void ByteBuffer::getBytes(void *buffer, size_t count) +void ByteBuffer::getBytes(void *buffer, uint32_t count) { - LOG_DEBUG3("getBytes(%p, %" PRIu64 ")", buffer, count); const char *v = getBufferAtPos(); incPos(count); memcpy(buffer, v, count); } -void ByteBuffer::putBytes(const void *buf, size_t count) { - LOG_DEBUG3("putBytes(%p, %" PRIu64 ")", buf, count); - if (__builtin_expect(getRemaining() < count, 0)) { - throwOutOfBounds(getRemaining(), sizeof(count)); - } else { - memcpy(getBufferAtPos(), buf, count); - incPosNoCheck(count); - } -} -std::string ByteBuffer::toString() { - std::ostringstream ost; - StringUtil::printAsHex(ost, getBuffer(), getLength()); - return ost.str(); -} - -void ByteBuffer::swap(ByteBuffer& other) { - LOG_DEBUG2("swap(%p)", &other); - std::swap(_bufHolder, other._bufHolder); - std::swap(_buffer, other._buffer); - std::swap(_len, other._len); - std::swap(_pos, other._pos); - std::swap(_limit, other._limit); -} - -void ByteBuffer::cleanUp() { - LOG_DEBUG1("cleanUp()"); - if (_bufHolder) { - _bufHolder->subRef(); - _bufHolder = NULL; - } else { - Alloc().swap(_ownedBuffer); - } - _buffer = NULL; -} } // document diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h index 6467e6d8bf0..4c367c0f143 100644 --- a/document/src/vespa/document/util/bytebuffer.h +++ b/document/src/vespa/document/util/bytebuffer.h @@ -15,7 +15,6 @@ #pragma once #include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/referencecounter.h> namespace document { @@ -23,19 +22,14 @@ class ByteBuffer { public: typedef std::unique_ptr<ByteBuffer> UP; - /** - * Creates a byte buffer with no underlying buffer. - * Use set() to set the buffer. - */ - ByteBuffer(); ByteBuffer(const ByteBuffer &); - ByteBuffer& operator=(const ByteBuffer &); + ByteBuffer& operator=(const ByteBuffer &) = delete; + ByteBuffer(ByteBuffer &&) = default; + ByteBuffer& operator=(ByteBuffer &&) = default; - ~ByteBuffer(); - - /** Allocates buffer with len bytes. */ - ByteBuffer(size_t len); + ByteBuffer() : ByteBuffer(nullptr, 0) { } + ~ByteBuffer() = default; /** * Create a buffer with the given content. @@ -43,7 +37,12 @@ public: * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(const char* buffer, size_t len); + ByteBuffer(const char* buffer, uint32_t len) + : _buffer(const_cast<char *>(buffer)), + _len(len), + _pos(0), + _ownedBuffer() + { } /** * Create a buffer with the given content. @@ -51,22 +50,8 @@ public: * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(vespalib::alloc::Alloc buffer, size_t len); - - /** - * Sets the buffer pointed to by this buffer. Allows for multiple - * usages of the same ByteBuffer. - */ - void set(const char* buffer, size_t len) { - cleanUp(); - _buffer = const_cast<char*>(buffer); - _len=len; - _limit=len; - _pos=0; - } - - /** Clear this buffer, and set free the underlying BufferHolder. */ - void reset() { set(NULL, 0); } + ByteBuffer(vespalib::alloc::Alloc buffer, uint32_t len); + ByteBuffer(std::unique_ptr<vespalib::alloc::Alloc> buffer, uint32_t len); /** * Creates a ByteBuffer object from another buffer. allocates @@ -75,62 +60,28 @@ public: * @param buffer The buffer to copy. * @param len The length of the buffer. * - * @return Returns a newly created bytebuffer object, or NULL - * if buffer was NULL, or len was <=0. + * @return Returns a newly created bytebuffer object, or nullptr + * if buffer was nullptr, or len was <=0. */ - static ByteBuffer* copyBuffer(const char* buffer, size_t len); - - std::unique_ptr<ByteBuffer> sliceCopy() const; - - /** - * @throws BufferOutOfBoundsException If faulty range is given. - */ - void sliceFrom(const ByteBuffer& buf, size_t from, size_t to); + static ByteBuffer copyBuffer(const char* buffer, uint32_t len); /** @return Returns the buffer pointed to by this object (at position 0) */ - char* getBuffer() const { return _buffer; } + const char* getBuffer() const { return _buffer; } /** @return Returns the length of the buffer pointed to by this object. */ - size_t getLength() const { return _len; } - - /** - * Adjust the length of the buffer. Only sane to shorten it, as you do not - * know what is ahead. - */ - void setLength(size_t len) { _len = len; } + uint32_t getLength() const { return _len; } /** @return Returns a pointer to the current position in the buffer. */ - char* getBufferAtPos() const { return _buffer + _pos; } + const char* getBufferAtPos() const { return _buffer + _pos; } /** @return Returns the index of the current position in the buffer. */ - size_t getPos() const { return _pos; } - - /** @return Returns the limit. */ - size_t getLimit() const { return _limit; } + uint32_t getPos() const { return _pos; } /** * @return Returns the number of bytes remaining in the buffer - that is, - * getLimit()-getPos(). + * getLength()-getPos(). */ - size_t getRemaining() const { return _limit-_pos; } - - /** - * Changes the position in the buffer. - * - * @throws BufferOutOfBoundsException; - */ - void setPos(size_t pos); - - /** - * Sets the buffer limit. - * - * @param limit The new limit. - * @return True if the limit is legal (less than the length) - * @throws BufferOutOfBoundsException; - */ - void setLimit(size_t limit); - size_t forceValgrindStart2Pos() const __attribute__ ((noinline)); - size_t forceValgrindPos2Lim() const __attribute__ ((noinline)); + uint32_t getRemaining() const { return _len -_pos; } /** * Moves the position in the buffer. @@ -138,267 +89,37 @@ public: * @param pos The number of bytes to move the position. The new position * will be oldPos + pos. This is the same as doing * setPos(getPos()+pos) - * @return True if the position could be moved (it was inside the limit - * of the buffer). * @throws BufferOutOfBoundsException; */ - void incPos(size_t pos); - - void incPosNoCheck(size_t pos) { - _pos += pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif - } - - /** - * Resets pos to 0, and sets limit to old pos. Use this before reading - * from a buffer you have written to - */ - void flip() { - _limit = _pos; - _pos = 0; - } + void incPos(uint32_t pos); - /** - * Sets pos to 0 and limit to length. Use this to start writing from the - * start of the buffer. - */ - void clear() { - _pos=0; - _limit=_len; - } - - void getNumericNetwork(uint8_t & v) { getNumeric(v); } void getNumeric(uint8_t & v); - void putNumericNetwork(uint8_t v) { putNumeric(v); } - void putNumeric(uint8_t v); void getNumericNetwork(int16_t & v); - void getNumeric(int16_t & v); - void putNumericNetwork(int16_t v); - void putNumeric(int16_t v); void getNumericNetwork(int32_t & v); - void getNumeric(int32_t & v); - void putNumericNetwork(int32_t v); - void putNumeric(int32_t v); - void getNumericNetwork(float & v); - void getNumeric(float & v); - void putNumericNetwork(float v); - void putNumeric(float v); + void getNumericNetwork(int64_t & v); void getNumeric(int64_t& v); - void putNumericNetwork(int64_t v); - void putNumeric(int64_t v); void getNumericNetwork(double & v); - void getNumeric(double& v); - void putNumericNetwork(double v); - void putNumeric(double v); + void getChar(char & val) { unsigned char t;getByte(t); val=t; } void getByte(uint8_t & v) { getNumeric(v); } - void putByte(uint8_t v) { putNumeric(v); } void getShortNetwork(int16_t & v) { getNumericNetwork(v); } - void getShort(int16_t & v) { getNumeric(v); } - void putShortNetwork(int16_t v) { putNumericNetwork(v); } - void putShort(int16_t v) { putNumeric(v); } void getIntNetwork(int32_t & v) { getNumericNetwork(v); } - void getInt(int32_t & v) { getNumeric(v); } - void putIntNetwork(int32_t v) { putNumericNetwork(v); } - void putInt(int32_t v) { putNumeric(v); } - void getFloatNetwork(float & v) { getNumericNetwork(v); } - void getFloat(float & v) { getNumeric(v); } - void putFloatNetwork(float v) { putNumericNetwork(v); } - void putFloat(float v) { putNumeric(v); } void getLongNetwork(int64_t & v) { getNumericNetwork(v); } void getLong(int64_t& v) { getNumeric(v); } - void putLongNetwork(int64_t v) { putNumericNetwork(v); } - void putLong(int64_t v) { putNumeric(v); } void getDoubleNetwork(double & v) { getNumericNetwork(v); } - void getDouble(double& v) { getNumeric(v); } - void putDoubleNetwork(double v) { putNumericNetwork(v); } - void putDouble(double v) { putNumeric(v); } - - private: - void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline,noreturn)); - uint8_t peekByte() const { return *getBufferAtPos(); } - -#if defined(__i386__) || defined(__x86_64__) - - template<typename T> - void putDoubleLongNetwork(T val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - putByte(data[i]); - } - } - - template<typename T> - void getDoubleLongNetwork(T &val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } - - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - getByte(data[i]); - } - } -#else - #error "getDoubleLongNetwork is undefined for this arcitecture" -#endif - - public: - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - */ - void putInt2_4_8Bytes(int64_t number) { - putInt2_4_8Bytes(number, 0); - } - - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - * @param len if non-zero, force writing number using len bytes, possibly - * with truncation - */ - void putInt2_4_8Bytes(int64_t number, size_t len); - - /** - * Reads a 62-bit positive integer from the buffer, which was written using - * 2, 4, or 8 bytes. - * - * @param v the integer read - */ - void getInt2_4_8Bytes(int64_t & v); - - /** - * Computes the size used for storing the given integer using 2, 4 or 8 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 2, 4 or 8 - */ - static size_t getSerializedSize2_4_8Bytes(int64_t number); - - /** - * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_2_4Bytes(int32_t number); - - /** - * Reads a 30-bit positive integer from the buffer, which was written using - * 1, 2, or 4 bytes. - * - * @param v the integer read - */ - void getInt1_2_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1, 2 or 4 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1, 2 or 4 - */ - static size_t getSerializedSize1_2_4Bytes(int32_t number); - - /** - * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_4Bytes(int32_t number); - - /** - * Reads a 31-bit positive integer from the buffer, which was written using - * 1 or 4 bytes. - * - * @param v the integer read - */ - void getInt1_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1 or 4 bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1 or 4 - */ - static size_t getSerializedSize1_4Bytes(int32_t number); - - /** - * Writes a 8 bit integer to the buffer at the current position, and - * increases the positition accordingly. - * - * @param val the int to store - * @return True if the value could be stored, false if end of buffer is - * reached - */ - void getChar(char & val) { unsigned char t;getByte(t); val=t; } - void putChar(char val) { putByte(static_cast<unsigned char>(val)); } - - /** - * Reads the given number of bytes into the given pointer, and updates the - * positition accordingly - * - * @param buffer where to store the bytes - * @param count number of bytes to read - * @return True if all the bytes could be read, false if end of - * buffer is reached - */ - void getBytes(void *buffer, size_t count); - - /** - * Writes the given number of bytes into the ByteBuffer at the current - * position, and updates the positition accordingly - * - * @param buf the bytes to store - * @param count number of bytes to store - */ - void putBytes(const void *buf, size_t count); - - /** Debug */ - void dump() const; - - class BufferHolder : public vespalib::ReferenceCounter - { - private: - BufferHolder(const BufferHolder &); - BufferHolder& operator=(const BufferHolder &); - - public: - BufferHolder(vespalib::alloc::Alloc buffer); - virtual ~BufferHolder(); - - vespalib::alloc::Alloc _buffer; - }; - - ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit); - - void set(BufferHolder* buf, size_t pos, size_t len, size_t limit); + void getBytes(void *buffer, uint32_t count); private: - char * _buffer; - size_t _len; - size_t _pos; - size_t _limit; - mutable BufferHolder * _bufHolder; - vespalib::alloc::Alloc _ownedBuffer; -public: - - std::string toString(); + template<typename T> + void getDoubleLongNetwork(T &val); - void swap(ByteBuffer& other); + void incPosNoCheck(uint32_t pos) { _pos += pos; } - void cleanUp(); + const char * _buffer; + uint32_t _len; + uint32_t _pos; + std::unique_ptr<vespalib::alloc::Alloc> _ownedBuffer; }; } // document diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp deleted file mode 100644 index 32c7bef90f0..00000000000 --- a/document/src/vespa/document/util/serializable.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serializable.h" -#include "bufferexceptions.h" -#include "serializableexceptions.h" -#include "bytebuffer.h" - -namespace document { - -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable); -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Deserializable, Serializable); -VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(SerializeException); - -std::unique_ptr<ByteBuffer> Serializable::serialize() const -{ - size_t len = getSerializedSize(); - std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(len)); - serialize(*retVal.get()); - return retVal; -} - -DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -DeserializeException::DeserializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -SerializeException::SerializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -SerializeException::SerializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -void -Serializable::serialize(ByteBuffer& buffer) const { - int pos = buffer.getPos(); - try{ - onSerialize(buffer); - } catch (...) { - buffer.setPos(pos); - throw; - } -} - -void -Deserializable::deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) { - int pos = buffer.getPos(); - try { - onDeserialize(repo, buffer); - } catch (const DeserializeException &) { - buffer.setPos(pos); - throw; - } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); - throw; - } -} -} diff --git a/document/src/vespa/document/util/serializable.h b/document/src/vespa/document/util/serializable.h deleted file mode 100644 index 8818feceb9a..00000000000 --- a/document/src/vespa/document/util/serializable.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file serializable.h - * @ingroup document - * - * @brief Interfaces to be used for serializing of objects. - * - * @author Thomas F. Gundersen, H�kon Humberset - * @date 2004-03-15 - * @version $Id$ - */ - -#pragma once - -#include <vespa/vespalib/objects/cloneable.h> -#include <vespa/vespalib/objects/identifiable.h> - -#include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/identifiableid.h> - -namespace document { -class DocumentTypeRepo; - -/** - * Base class for classes that can be converted into a bytestream, - * normally used later to create a similar instance. - */ - -class Serializable : public vespalib::Identifiable -{ -protected: - virtual void onSerialize(ByteBuffer& buffer) const = 0; -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Serializable); - - virtual ~Serializable() {} - - /** - * @return An upper limit to how many bytes serialization of this instance - * need, providing instance is not altered before serialization. - */ - virtual size_t getSerializedSize() const = 0; - - /** - * Serializes the instance into the buffer given. Use getSerializedSize() - * before calling this method to be sure buffer is big enough. - * On success, the given buffers position will be just past the serialized - * version of this instance, on failure, position will be reset to whatever - * it was prior to calling this function. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - void serialize(ByteBuffer& buffer) const; - - /** - * Creates a bytebuffer with enough space to serialize this instance - * and serialize this instance into it. - * - * @return The created bytebuffer, positioned after the serialization. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - std::unique_ptr<ByteBuffer> serialize() const; -}; - -/** - * Base class for instances that can be overwritten from a bytestream, - * given that the bytestream is created from a similar instance. - */ -class Deserializable : public vespalib::Cloneable, public Serializable -{ -protected: - virtual void onDeserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) = 0; - -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Deserializable); - virtual ~Deserializable() {} - - /** - * Overwrite this object with the object represented by the given - * bytestream. On success, buffer will be positioned after the bytestream - * representing the instance we've just deserialized, on failure, bytebuffer - * will be pointing to where it was pointing before calling this function. - * - * @throw DeserializeException If read data doesn't represent a legal object - * of this type. - * @throw BufferOutOfBoundsException If instance wants to read more data - * than is available in the buffer. - */ - void deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer); -}; - -} - diff --git a/document/src/vespa/document/util/serializableexceptions.cpp b/document/src/vespa/document/util/serializableexceptions.cpp new file mode 100644 index 00000000000..e80e38015e8 --- /dev/null +++ b/document/src/vespa/document/util/serializableexceptions.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializableexceptions.h" + +namespace document { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); + +DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, location, 1) +{ +} + +DeserializeException::DeserializeException( + const vespalib::string& msg, const vespalib::Exception& cause, + const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) +{ +} + +} diff --git a/document/src/vespa/document/util/serializableexceptions.h b/document/src/vespa/document/util/serializableexceptions.h index fcfed810bfc..1b692aa27b7 100644 --- a/document/src/vespa/document/util/serializableexceptions.h +++ b/document/src/vespa/document/util/serializableexceptions.h @@ -24,12 +24,4 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(DeserializeException) }; -class SerializeException : public vespalib::IoException { -public: - SerializeException(const vespalib::string& msg, const vespalib::string& location = ""); - SerializeException(const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location = ""); - VESPA_DEFINE_EXCEPTION_SPINE(SerializeException) -}; - } diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index 8d76cf5974e..c7bb1015e02 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -218,7 +218,7 @@ Messages60Test::testDocumentListMessage() DocumentListMessage tmp(document::BucketId(16, 1234)); tmp.getDocuments().push_back(entry); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)69, serialize("DocumentListMessage", tmp)); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 69ul, serialize("DocumentListMessage", tmp)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { mbus::Routable::UP obj = deserialize("DocumentListMessage", DocumentProtocol::MESSAGE_DOCUMENTLIST, lang); @@ -407,7 +407,7 @@ Messages60Test::testPutDocumentMessage() for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { auto routableUp = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); - if (EXPECT_TRUE(routableUp.get() != nullptr)) { + if (EXPECT_TRUE(routableUp)) { auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp); EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName()); diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index e8cc7a03e94..a2e8219e916 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -48,7 +48,7 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, std::vector<vespalib::VersionSpecification> from6 = { version6 }; // Add 6.x serialization - putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory()), from6); putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories60::DestroyVisitorMessageFactory()), from6); putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories60::DocumentListMessageFactory(*_repo)), from6); putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories60::DocumentSummaryMessageFactory()), from6); @@ -56,7 +56,7 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories60::GetBucketListMessageFactory()), from6); putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories60::GetBucketStateMessageFactory()), from6); putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::GetDocumentMessageFactory()), from6); - putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorMessageFactory()), from6); putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::PutDocumentMessageFactory(*_repo)), from6); putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories60::QueryResultMessageFactory()), from6); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::RemoveDocumentMessageFactory()), from6); diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 79f7d7c0ccc..5582c0ea153 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -20,7 +20,6 @@ namespace documentapi { class LoadTypeSet; class RoutingPolicyRepository; class RoutableRepository; -class SystemState; class IRoutingPolicyFactory; class IRoutableFactory; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp index 718611dfc04..6c8394b1b4c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp @@ -15,12 +15,11 @@ DocumentState::DocumentState(const DocumentState& o) : _gid(o._gid), _timestamp(o._timestamp), _removeEntry(o._removeEntry) { if (o._docId.get() != 0) { - _docId.reset(new document::DocumentId(*o._docId)); + _docId = std::make_unique<document::DocumentId>(*o._docId); } } -DocumentState::DocumentState(const document::DocumentId& id, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::DocumentId& id, uint64_t timestamp, bool removeEntry) : _docId(new document::DocumentId(id)), _gid(_docId->getGlobalId()), _timestamp(timestamp), @@ -28,8 +27,7 @@ DocumentState::DocumentState(const document::DocumentId& id, { } -DocumentState::DocumentState(const document::GlobalId& gid, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::GlobalId& gid, uint64_t timestamp, bool removeEntry) : _gid(gid), _timestamp(timestamp), _removeEntry(removeEntry) {} DocumentState::DocumentState(document::ByteBuffer& buf) @@ -39,10 +37,10 @@ DocumentState::DocumentState(document::ByteBuffer& buf) buf.getByte(hasDocId); if (hasDocId) { vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); - _docId.reset(new document::DocumentId(stream)); + _docId = std::make_unique<document::DocumentId>(stream); buf.incPos(stream.rp()); } - char* gid = buf.getBufferAtPos(); + const char* gid = buf.getBufferAtPos(); buf.incPos(document::GlobalId::LENGTH); _gid.set(gid); buf.getLongNetwork((int64_t&) _timestamp); @@ -55,8 +53,8 @@ DocumentState& DocumentState::operator=(const DocumentState& other) { _docId.reset(); - if (other._docId.get() != 0) { - _docId.reset(new document::DocumentId(*other._docId)); + if (other._docId) { + _docId = std::make_unique<document::DocumentId>(*other._docId); } _gid = other._gid; _timestamp = other._timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp index 6753d269ad6..2dcaccbd861 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp @@ -21,12 +21,12 @@ PutDocumentMessage::PutDocumentMessage(document::Document::SP document) : setDocument(std::move(document)); } -PutDocumentMessage::~PutDocumentMessage() {} +PutDocumentMessage::~PutDocumentMessage() = default; DocumentReply::UP PutDocumentMessage::doCreateReply() const { - return DocumentReply::UP(new WriteDocumentReply(DocumentProtocol::REPLY_PUTDOCUMENT)); + return std::make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); } bool diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp index 411a7237cb5..08d631dc44b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp @@ -4,11 +4,7 @@ namespace documentapi { -QueryResultMessage::QueryResultMessage() : - VisitorMessage(), - _searchResult(), - _summary() -{} +QueryResultMessage::QueryResultMessage() = default; QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary) : VisitorMessage(), @@ -16,7 +12,7 @@ QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, cons _summary(summary) {} -QueryResultMessage::~QueryResultMessage() {} +QueryResultMessage::~QueryResultMessage() = default; DocumentReply::UP QueryResultMessage::doCreateReply() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h index 239ce6fefd5..6324e2664e4 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h @@ -25,7 +25,7 @@ public: * Constructs a new search result message for deserialization. */ QueryResultMessage(); - ~QueryResultMessage(); + ~QueryResultMessage() override; /** * Constructs a new search result message for the given search result. diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index 43ad30ea24f..453e93fd7eb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitor.h" -#include <climits> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> +#include <climits> using document::FixedBucketSpaces; @@ -56,7 +59,7 @@ CreateVisitorMessage::~CreateVisitorMessage() = default; DocumentReply::UP CreateVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); + return std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); } uint32_t @@ -65,11 +68,7 @@ CreateVisitorMessage::getType() const return DocumentProtocol::MESSAGE_CREATEVISITOR; } -DestroyVisitorMessage::DestroyVisitorMessage() : - DocumentMessage(), - _instanceId() -{ -} +DestroyVisitorMessage::DestroyVisitorMessage() = default; DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : DocumentMessage(), @@ -77,13 +76,12 @@ DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : { } -DestroyVisitorMessage::~DestroyVisitorMessage() { -} +DestroyVisitorMessage::~DestroyVisitorMessage() = default; DocumentReply::UP DestroyVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_DESTROYVISITOR)); + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_DESTROYVISITOR); } uint32_t @@ -104,20 +102,13 @@ CreateVisitorReply::CreateVisitorReply(uint32_t type) : { } -VisitorInfoMessage::VisitorInfoMessage() : - VisitorMessage(), - _finishedBuckets(), - _errorMessage() -{ -} - -VisitorInfoMessage::~VisitorInfoMessage() { -} +VisitorInfoMessage::VisitorInfoMessage() = default; +VisitorInfoMessage::~VisitorInfoMessage() = default; DocumentReply::UP VisitorInfoMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); } uint32_t @@ -126,11 +117,7 @@ VisitorInfoMessage::getType() const return DocumentProtocol::MESSAGE_VISITORINFO; } -MapVisitorMessage::MapVisitorMessage() : - _data() -{ - // empty -} +MapVisitorMessage::MapVisitorMessage() = default; uint32_t MapVisitorMessage::getApproxSize() const @@ -141,7 +128,7 @@ MapVisitorMessage::getApproxSize() const DocumentReply::UP MapVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); } uint32_t MapVisitorMessage::getType() const @@ -149,60 +136,37 @@ uint32_t MapVisitorMessage::getType() const return DocumentProtocol::MESSAGE_MAPVISITOR; } -DocumentListMessage::Entry::Entry() -{ - // empty -} +DocumentListMessage::Entry::Entry() = default; -DocumentListMessage::Entry::Entry(int64_t timestamp, - document::Document::SP doc, - bool removeEntry) : +DocumentListMessage::Entry::Entry(int64_t timestamp, document::Document::SP doc, bool removeEntry) : _timestamp(timestamp), - _document(doc), + _document(std::move(doc)), _removeEntry(removeEntry) -{ - // empty -} +{ } -DocumentListMessage::Entry::Entry(const Entry& other) : - _timestamp(other._timestamp), - _document(other._document), - _removeEntry(other._removeEntry) -{ - // empty -} +DocumentListMessage::Entry::Entry(const Entry& other) = default; -DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buf) +DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, document::ByteBuffer& buf) { buf.getLongNetwork(_timestamp); - _document.reset(new document::Document(repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + _document = std::make_unique<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); uint8_t b; buf.getByte(b); _removeEntry = b>0; } void -DocumentListMessage::Entry::serialize(document::ByteBuffer& buf) const +DocumentListMessage::Entry::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putLongNetwork(_timestamp); - _document->serialize(buf); + buf.putLong(_timestamp); + vespalib::nbostream nbo = _document->serialize(); + buf.putBytes(nbo.data(), nbo.size()); buf.putByte(_removeEntry ? 1 : 0); } -uint32_t -DocumentListMessage::Entry::getSerializedSize() const -{ - return sizeof(int64_t) + sizeof(uint8_t) - + _document->serialize()->getLength(); -} - -DocumentListMessage::DocumentListMessage() : - _bucketId(), - _documents() -{ - // empty -} +DocumentListMessage::DocumentListMessage() = default; DocumentListMessage::DocumentListMessage(document::BucketId bid) : _bucketId(bid), @@ -214,7 +178,7 @@ DocumentListMessage::DocumentListMessage(document::BucketId bid) : DocumentReply::UP DocumentListMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index b18a4e985f3..f47fa48bd80 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -52,7 +52,7 @@ public: const string& instanceId, const string& controlDestination, const string& dataDestination); - ~CreateVisitorMessage(); + ~CreateVisitorMessage() override; const string& getLibraryName() const { return _libName; } void setLibraryName(const string& value) { _libName = value; } @@ -250,7 +250,7 @@ public: const document::Document::SP& getDocument() { return _document; } bool isRemoveEntry() { return _removeEntry; } - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: int64_t _timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 797f55120fc..7bae7dd7e77 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -20,7 +20,7 @@ namespace documentapi { bool RoutableFactories60::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const { - const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + const auto &msg = static_cast<const DocumentMessage&>(obj); out.putByte(msg.getPriority()); out.putInt(msg.getLoadType().getId()); return doEncode(msg, out); @@ -93,7 +93,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer msg->setVisitRemoves(decodeBoolean(buf)); msg->setFieldSet(decodeString(buf)); msg->setVisitInconsistentBuckets(decodeBoolean(buf)); - msg->getParameters().deserialize(_repo, buf); + msg->getParameters().deserialize(buf); msg->setVisitorDispatcherVersion(50); decodeInt(buf); // Unused legacy visitor ordering msg->setMaxBucketsPerVisitor(decodeInt(buf)); @@ -105,7 +105,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + const auto &msg = static_cast<const CreateVisitorMessage&>(obj); buf.putString(msg.getLibraryName()); buf.putString(msg.getInstanceId()); @@ -126,10 +126,8 @@ RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); - int len = msg.getParameters().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getParameters().serialize(dbuf); + + msg.getParameters().serialize(buf); buf.putInt(0); // Unused legacy visitor ordering buf.putInt(msg.getMaxBucketsPerVisitor()); @@ -184,7 +182,7 @@ RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); + const auto &reply = static_cast<const CreateVisitorReply&>(obj); buf.putLong(reply.getLastBucket().getRawId()); buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); @@ -209,19 +207,14 @@ RoutableFactories60::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, } DocumentReply::UP -RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& ) const { - (void) buf; - return DocumentReply::UP(new DocumentIgnoredReply()); + return std::make_unique<DocumentIgnoredReply>(); } bool -RoutableFactories60::DocumentIgnoredReplyFactory::doEncode( - const DocumentReply& obj, - vespalib::GrowableByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doEncode(const DocumentReply&, vespalib::GrowableByteBuffer& ) const { - (void) obj; - (void) buf; return true; } @@ -243,15 +236,12 @@ RoutableFactories60::DocumentListMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); + const auto &msg = static_cast<const DocumentListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putInt(msg.getDocuments().size()); for (const auto & document : msg.getDocuments()) { - int len = document.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - document.serialize(dbuf); + document.serialize(buf); } return true; @@ -283,11 +273,7 @@ bool RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - - int32_t len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + msg.serialize(buf); return true; } @@ -322,7 +308,7 @@ RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); + const auto &msg = static_cast<const EmptyBucketsMessage&>(obj); buf.putInt(msg.getBucketIds().size()); for (const auto & bucketId : msg.getBucketIds()) { @@ -367,7 +353,7 @@ RoutableFactories60::GetBucketListMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); + const auto &msg = static_cast<const GetBucketListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return encodeBucketSpace(msg.getBucketSpace(), buf); } @@ -392,7 +378,7 @@ RoutableFactories60::GetBucketListReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); + const auto &reply = static_cast<const GetBucketListReply&>(obj); const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); buf.putInt(buckets.size()); @@ -417,7 +403,7 @@ RoutableFactories60::GetBucketStateMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); + const auto &msg = static_cast<const GetBucketStateMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return true; } @@ -452,14 +438,12 @@ RoutableFactories60::GetBucketStateReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories60::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const { - return DocumentMessage::UP( - new GetDocumentMessage(decodeDocumentId(buf), - decodeString(buf))); + document::DocumentId docId = decodeDocumentId(buf); + return std::make_unique<GetDocumentMessage>(docId, decodeString(buf)); } bool -RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, - vespalib::GrowableByteBuffer &buf) const +RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); @@ -468,6 +452,17 @@ RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage & return true; } +namespace { + +std::shared_ptr<document::Document> +decodeDocument(const document::DocumentTypeRepo & repo, document::ByteBuffer & buf) { + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + auto doc = std::make_shared<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); + return doc; +} + +} DocumentReply::UP RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { @@ -476,7 +471,7 @@ RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf bool hasDocument = decodeBoolean(buf); document::Document * document = nullptr; if (hasDocument) { - auto doc = std::make_shared<document::Document>(_repo, buf); + auto doc = decodeDocument(_repo, buf); document = doc.get(); reply->setDocument(std::move(doc)); } @@ -509,7 +504,7 @@ DocumentMessage::UP RoutableFactories60::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<MapVisitorMessage>(); - msg->getData().deserialize(_repo, buf); + msg->getData().deserialize(buf); return msg; } @@ -517,11 +512,7 @@ bool RoutableFactories60::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); - - int32_t len = msg.getData().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getData().serialize(dbuf); + msg.getData().serialize(buf); return true; } @@ -540,7 +531,7 @@ RoutableFactories60::MapVisitorReplyFactory::doEncode(const DocumentReply &, ves void RoutableFactories60::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocument(make_shared<document::Document>(_repo, buf)); + msg.setDocument(decodeDocument(_repo, buf)); msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); } @@ -564,9 +555,6 @@ RoutableFactories60::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf { auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); reply->setHighestModificationTimestamp(decodeLong(buf)); - - // Doing an explicit move here to force converting result to an rvalue. - // This is done automatically in GCC >= 5. return reply; } @@ -656,12 +644,8 @@ RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); - - int len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + const auto & msg = static_cast<const SearchResultMessage&>(obj); + msg.serialize(buf); return true; } @@ -679,13 +663,10 @@ RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); + const auto &msg = static_cast<const QueryResultMessage&>(obj); - int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getSearchResult().serialize(dbuf); - msg.getDocumentSummary().serialize(dbuf); + msg.getSearchResult().serialize(buf); + msg.getDocumentSummary().serialize(buf); return true; } @@ -740,7 +721,7 @@ RoutableFactories60::StatBucketMessageFactory::doDecode(document::ByteBuffer &bu bool RoutableFactories60::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); + const auto &msg = static_cast<const StatBucketMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putString(msg.getDocumentSelection()); @@ -758,7 +739,7 @@ RoutableFactories60::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) bool RoutableFactories60::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); + const auto &reply = static_cast<const StatBucketReply&>(obj); buf.putString(reply.getResults()); return true; } @@ -789,7 +770,9 @@ RoutableFactories60::StatDocumentReplyFactory::doEncode(const DocumentReply &, v void RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, stream)); + buf.incPos(stream.rp()); msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); @@ -798,7 +781,7 @@ RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMess bool RoutableFactories60::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); + const auto &msg = static_cast<const UpdateDocumentMessage&>(obj); vespalib::nbostream stream; msg.getDocumentUpdate().serializeHEAD(stream); @@ -822,7 +805,7 @@ RoutableFactories60::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); + const auto &reply = static_cast<const UpdateDocumentReply&>(obj); buf.putBoolean(reply.getWasFound()); buf.putLong(reply.getHighestModificationTimestamp()); return true; @@ -848,7 +831,7 @@ RoutableFactories60::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); + const auto &msg = static_cast<const VisitorInfoMessage&>(obj); buf.putInt(msg.getFinishedBuckets().size()); for (const auto & bucketId : msg.getFinishedBuckets()) { @@ -883,7 +866,7 @@ RoutableFactories60::WrongDistributionReplyFactory::doDecode(document::ByteBuffe bool RoutableFactories60::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); + const auto &reply = static_cast<const WrongDistributionReply&>(obj); buf.putString(reply.getSystemState()); return true; } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h index 75404f1dea1..0a997f3ffd9 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h @@ -130,7 +130,6 @@ public: // //////////////////////////////////////////////////////////////////////////////// class CreateVisitorMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; @@ -138,7 +137,7 @@ public: virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; virtual string decodeBucketSpace(document::ByteBuffer&) const; public: - CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + CreateVisitorMessageFactory() : DocumentMessageFactory() {} }; class CreateVisitorReplyFactory : public DocumentReplyFactory { protected: @@ -228,12 +227,11 @@ public: GetDocumentReplyFactory(const document::DocumentTypeRepo &r) : _repo(r) {} }; class MapVisitorMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: - MapVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + MapVisitorMessageFactory() : DocumentMessageFactory() {} }; class MapVisitorReplyFactory : public DocumentReplyFactory { protected: diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index 872634bf53b..dd6ccafeab5 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -67,6 +67,10 @@ search book { attribute: tensor(x{}) } } + + import field ref.dummy as my_dummy {} + import field ref.foo as my_foo {} + field sw1 type float { } field didinteger type array<int> { diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd index 99a50fdf8ea..50b8e76cf5a 100644 --- a/documentgen-test/etc/complex/parent.sd +++ b/documentgen-test/etc/complex/parent.sd @@ -4,7 +4,10 @@ search parent { document parent { field dummy type string { - + indexing: attribute + } + field foo type string { + indexing: attribute } } } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 6339416d007..29bee2e9e3e 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1028,5 +1028,14 @@ public class DocumentGenPluginTest { assertTrue(book.getDataType().fieldSetAll().contains(posZcurve)); assertTrue(book.getDataType().getFields().contains(posZcurve)); } + + @Test + public void imported_fields_are_enumerated_in_document_type() { + var docType = getBook().getDataType(); + assertEquals(2, docType.getImportedFieldNames().size()); + assertTrue(docType.hasImportedField("my_dummy")); + assertTrue(docType.hasImportedField("my_foo")); + assertFalse(docType.hasImportedField("some_field_that_does_not_exist")); + } } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index b1212650a3f..febb254c53e 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -33,6 +33,7 @@ vespa_define_module( src/tests/tensor/dense_generic_join src/tests/tensor/dense_inplace_join_function src/tests/tensor/dense_inplace_map_function + src/tests/tensor/dense_matmul_function src/tests/tensor/dense_remove_dimension_optimizer src/tests/tensor/dense_replace_type_function src/tests/tensor/dense_tensor_create_function diff --git a/eval/src/tests/ann/CMakeLists.txt b/eval/src/tests/ann/CMakeLists.txt index 05256d19f00..52b4d675d9c 100644 --- a/eval/src/tests/ann/CMakeLists.txt +++ b/eval/src/tests/ann/CMakeLists.txt @@ -9,3 +9,13 @@ vespa_add_executable(eval_sift_benchmark_app DEPENDS vespaeval ) + +vespa_add_executable(eval_remove_bm_app + SOURCES + remove-bm.cpp + xp-annoy-nns.cpp + xp-hnswlike-nns.cpp + xp-lsh-nns.cpp + DEPENDS + vespaeval +) diff --git a/eval/src/tests/ann/nns-l2.h b/eval/src/tests/ann/nns-l2.h index dcad5f1bda6..857866ff73b 100644 --- a/eval/src/tests/ann/nns-l2.h +++ b/eval/src/tests/ann/nns-l2.h @@ -2,6 +2,7 @@ #pragma once #include <string.h> +#include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> template <typename T, size_t VLEN> diff --git a/eval/src/tests/ann/nns.h b/eval/src/tests/ann/nns.h index 79c1aac4379..ffe2882188e 100644 --- a/eval/src/tests/ann/nns.h +++ b/eval/src/tests/ann/nns.h @@ -67,3 +67,7 @@ make_rplsh_nns(uint32_t numDims, const DocVectorAccess<float> &dva); extern std::unique_ptr<NNS<float>> make_hnsw_nns(uint32_t numDims, const DocVectorAccess<float> &dva); + +extern +std::unique_ptr<NNS<float>> +make_hnsw_wrap(uint32_t numDims, const DocVectorAccess<float> &dva); diff --git a/eval/src/tests/ann/remove-bm.cpp b/eval/src/tests/ann/remove-bm.cpp new file mode 100644 index 00000000000..be010552ab8 --- /dev/null +++ b/eval/src/tests/ann/remove-bm.cpp @@ -0,0 +1,514 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/priority_queue.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdio.h> +#include <chrono> + +#define NUM_DIMS 960 +#define NUM_DOCS 250000 +#define NUM_DOCS_REMOVE 50000 +#define EFFECTIVE_DOCS (NUM_DOCS - NUM_DOCS_REMOVE) +#define NUM_Q 1000 + +#include "doc_vector_access.h" +#include "nns.h" +#include "for-sift-hit.h" +#include "for-sift-top-k.h" + +std::vector<TopK> bruteforceResults; +std::vector<float> tmp_v(NUM_DIMS); + +struct PointVector { + float v[NUM_DIMS]; + using ConstArr = vespalib::ConstArrayRef<float>; + operator ConstArr() const { return ConstArr(v, NUM_DIMS); } +}; + +static PointVector *aligned_alloc(size_t num) { + size_t sz = num * sizeof(PointVector); + double mega_bytes = sz / (1024.0*1024.0); + fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes); + char *mem = (char *)malloc(sz + 512); + mem += 512; + size_t val = (size_t)mem; + size_t unalign = val % 512; + mem -= unalign; + return reinterpret_cast<PointVector *>(mem); +} + +static PointVector *generatedQueries = aligned_alloc(NUM_Q); +static PointVector *generatedDocs = aligned_alloc(NUM_DOCS); + +struct DocVectorAdapter : public DocVectorAccess<float> +{ + vespalib::ConstArrayRef<float> get(uint32_t docid) const override { + ASSERT_TRUE(docid < NUM_DOCS); + return generatedDocs[docid]; + } +}; + +double computeDistance(const PointVector &query, uint32_t docid) { + const PointVector &docvector = generatedDocs[docid]; + return l2distCalc.l2sq_dist(query, docvector, tmp_v); +} + +void read_queries(std::string fn) { + int fd = open(fn.c_str(), O_RDONLY); + ASSERT_TRUE(fd > 0); + int d; + size_t rv; + fprintf(stderr, "reading %u queries from %s\n", NUM_Q, fn.c_str()); + for (uint32_t qid = 0; qid < NUM_Q; ++qid) { + rv = read(fd, &d, 4); + ASSERT_EQUAL(rv, 4u); + ASSERT_EQUAL(d, NUM_DIMS); + rv = read(fd, &generatedQueries[qid].v, NUM_DIMS*sizeof(float)); + ASSERT_EQUAL(rv, sizeof(PointVector)); + } + close(fd); +} + +void read_docs(std::string fn) { + int fd = open(fn.c_str(), O_RDONLY); + ASSERT_TRUE(fd > 0); + int d; + size_t rv; + fprintf(stderr, "reading %u doc vectors from %s\n", NUM_DOCS, fn.c_str()); + for (uint32_t docid = 0; docid < NUM_DOCS; ++docid) { + rv = read(fd, &d, 4); + ASSERT_EQUAL(rv, 4u); + ASSERT_EQUAL(d, NUM_DIMS); + rv = read(fd, &generatedDocs[docid].v, NUM_DIMS*sizeof(float)); + ASSERT_EQUAL(rv, sizeof(PointVector)); + } + close(fd); +} + +using TimePoint = std::chrono::steady_clock::time_point; +using Duration = std::chrono::steady_clock::duration; + +double to_ms(Duration elapsed) { + std::chrono::duration<double, std::milli> ms(elapsed); + return ms.count(); +} + +void read_data(std::string dir) { + TimePoint bef = std::chrono::steady_clock::now(); + read_queries(dir + "/gist_query.fvecs"); + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "read queries: %.3f ms\n", to_ms(aft - bef)); + bef = std::chrono::steady_clock::now(); + read_docs(dir + "/gist_base.fvecs"); + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "read docs: %.3f ms\n", to_ms(aft - bef)); +} + + +struct BfHitComparator { + bool operator() (const Hit &lhs, const Hit& rhs) const { + if (lhs.distance < rhs.distance) return false; + if (lhs.distance > rhs.distance) return true; + return (lhs.docid > rhs.docid); + } +}; + +class BfHitHeap { +private: + size_t _size; + vespalib::PriorityQueue<Hit, BfHitComparator> _priQ; +public: + explicit BfHitHeap(size_t maxSize) : _size(maxSize), _priQ() { + _priQ.reserve(maxSize); + } + ~BfHitHeap() {} + void maybe_use(const Hit &hit) { + if (_priQ.size() < _size) { + _priQ.push(hit); + } else if (hit.distance < _priQ.front().distance) { + _priQ.front() = hit; + _priQ.adjust(); + } + } + std::vector<Hit> bestHits() { + std::vector<Hit> result; + size_t i = _priQ.size(); + result.resize(i); + while (i-- > 0) { + result[i] = _priQ.front(); + _priQ.pop_front(); + } + return result; + } +}; + +TopK bruteforce_nns(const PointVector &query) { + TopK result; + BfHitHeap heap(result.K); + for (uint32_t docid = 0; docid < EFFECTIVE_DOCS; ++docid) { + const PointVector &docvector = generatedDocs[docid]; + double d = l2distCalc.l2sq_dist(query, docvector, tmp_v); + Hit h(docid, d); + heap.maybe_use(h); + } + std::vector<Hit> best = heap.bestHits(); + for (size_t i = 0; i < result.K; ++i) { + result.hits[i] = best[i]; + } + return result; +} + +void verifyBF(uint32_t qid) { + const PointVector &query = generatedQueries[qid]; + TopK &result = bruteforceResults[qid]; + double min_distance = result.hits[0].distance; + std::vector<double> all_c2; + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { + double dist = computeDistance(query, i); + if (dist < min_distance) { + fprintf(stderr, "WARN dist %.9g < mindist %.9g\n", dist, min_distance); + } + EXPECT_FALSE(dist+0.000001 < min_distance); + if (min_distance > 0.0) all_c2.push_back(dist / min_distance); + } + if (all_c2.size() != EFFECTIVE_DOCS) return; + std::sort(all_c2.begin(), all_c2.end()); + for (uint32_t idx : { 1, 3, 10, 30, 100, 300, 1000, 3000, EFFECTIVE_DOCS/2, EFFECTIVE_DOCS-1}) { + fprintf(stderr, "c2-factor[%u] = %.3f\n", idx, all_c2[idx]); + } +} + +using NNS_API = NNS<float>; + +#if 1 +TEST("require that HNSW via NNS api remove all works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_hnsw_nns(NUM_DIMS, adapter); + fprintf(stderr, "adding and removing all docs forward...\n"); + for (uint32_t i = 0; i < 1000; ++i) { + nns->addDoc(i); + } + for (uint32_t i = 0; i < 1000; ++i) { + nns->removeDoc(i); + } + fprintf(stderr, "adding and removing all docs reverse...\n"); + for (uint32_t i = 1000; i < 2000; ++i) { + nns->addDoc(i); + } + for (uint32_t i = 2000; i-- > 1000; ) { + nns->removeDoc(i); + } +} +#endif + +TEST("require that brute force works") { + TimePoint bef = std::chrono::steady_clock::now(); + fprintf(stderr, "generating %u brute force results\n", NUM_Q); + bruteforceResults.reserve(NUM_Q); + for (uint32_t cnt = 0; cnt < NUM_Q; ++cnt) { + const PointVector &query = generatedQueries[cnt]; + bruteforceResults.emplace_back(bruteforce_nns(query)); + } + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "timing for brute force: %.3f ms = %.3f ms per query\n", + to_ms(aft - bef), to_ms(aft - bef)/NUM_Q); + for (int cnt = 0; cnt < NUM_Q; cnt = (cnt+1)*2) { + verifyBF(cnt); + } +} + +bool reach_with_nns_1(NNS_API &nns, uint32_t docid) { + const PointVector &qv = generatedDocs[docid]; + vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS); + auto rv = nns.topK(1, query, 1); + if (rv.size() != 1) { + fprintf(stderr, "Result/A from query for %u is %zu hits\n", docid, rv.size()); + return false; + } + if (rv[0].docid != docid) { + if (rv[0].sq.distance != 0.0) + fprintf(stderr, "Expected/A to find %u but got %u with sq distance %.3f\n", + docid, rv[0].docid, rv[0].sq.distance); + } + return (rv[0].docid == docid || rv[0].sq.distance == 0.0); +} + +bool reach_with_nns_100(NNS_API &nns, uint32_t docid) { + const PointVector &qv = generatedDocs[docid]; + vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS); + auto rv = nns.topK(10, query, 100); + if (rv.size() != 10) { + fprintf(stderr, "Result/B from query for %u is %zu hits\n", docid, rv.size()); + } + if (rv[0].docid != docid) { + if (rv[0].sq.distance != 0.0) + fprintf(stderr, "Expected/B to find %u but got %u with sq distance %.3f\n", + docid, rv[0].docid, rv[0].sq.distance); + } + return (rv[0].docid == docid || rv[0].sq.distance == 0.0); +} + +bool reach_with_nns_1k(NNS_API &nns, uint32_t docid) { + const PointVector &qv = generatedDocs[docid]; + vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS); + auto rv = nns.topK(10, query, 1000); + if (rv.size() != 10) { + fprintf(stderr, "Result/C from query for %u is %zu hits\n", docid, rv.size()); + } + if (rv[0].docid != docid) { + if (rv[0].sq.distance != 0.0) + fprintf(stderr, "Expected/C to find %u but got %u with sq distance %.3f\n", + docid, rv[0].docid, rv[0].sq.distance); + } + return (rv[0].docid == docid || rv[0].sq.distance == 0.0); +} + +TopK find_with_nns(uint32_t sk, NNS_API &nns, uint32_t qid) { + TopK result; + const PointVector &qv = generatedQueries[qid]; + vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS); + auto rv = nns.topK(result.K, query, sk); + for (size_t i = 0; i < result.K; ++i) { + result.hits[i] = Hit(rv[i].docid, rv[i].sq.distance); + } + return result; +} + +void verify_nns_quality(uint32_t sk, NNS_API &nns, uint32_t qid) { + TopK perfect = bruteforceResults[qid]; + TopK result = find_with_nns(sk, nns, qid); + int recall = perfect.recall(result); + EXPECT_TRUE(recall > 40); + double sum_error = 0.0; + double c_factor = 1.0; + for (size_t i = 0; i < result.K; ++i) { + double factor = (result.hits[i].distance / perfect.hits[i].distance); + if (factor < 0.99 || factor > 25) { + fprintf(stderr, "hit[%zu] got distance %.3f, expected %.3f\n", + i, result.hits[i].distance, perfect.hits[i].distance); + } + sum_error += factor; + c_factor = std::max(c_factor, factor); + } + EXPECT_TRUE(c_factor < 1.5); + fprintf(stderr, "quality sk=%u: query %u: recall %d c2-factor %.3f avg c2: %.3f\n", + sk, qid, recall, c_factor, sum_error / result.K); +} + +void timing_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) { + for (uint32_t search_k : sk_list) { + TimePoint bef = std::chrono::steady_clock::now(); + for (int cnt = 0; cnt < NUM_Q; ++cnt) { + find_with_nns(search_k, nns, cnt); + } + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "timing for %s search_k=%u: %.3f ms = %.3f ms/q\n", + name, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q); + } +} + +void quality_nns(NNS_API &nns, std::vector<uint32_t> sk_list) { + for (uint32_t search_k : sk_list) { + for (int cnt = 0; cnt < NUM_Q; ++cnt) { + verify_nns_quality(search_k, nns, cnt); + } + } + uint32_t reached = 0; + for (uint32_t i = 0; i < 20000; ++i) { + if (reach_with_nns_1(nns, i)) ++reached; + } + fprintf(stderr, "Could reach %u of 20000 first documents with k=1\n", reached); + reached = 0; + for (uint32_t i = 0; i < 20000; ++i) { + if (reach_with_nns_100(nns, i)) ++reached; + } + fprintf(stderr, "Could reach %u of 20000 first documents with k=100\n", reached); + reached = 0; + for (uint32_t i = 0; i < 20000; ++i) { + if (reach_with_nns_1k(nns, i)) ++reached; + } + fprintf(stderr, "Could reach %u of 20000 first documents with k=1000\n", reached); +} + +void benchmark_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) { + fprintf(stderr, "trying %s indexing...\n", name); + +#if 0 + TimePoint bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.addDoc(EFFECTIVE_DOCS + i); + } + for (uint32_t i = 0; i < EFFECTIVE_DOCS - NUM_DOCS_REMOVE; ++i) { + nns.addDoc(i); + } + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.removeDoc(EFFECTIVE_DOCS + i); + nns.addDoc(EFFECTIVE_DOCS - NUM_DOCS_REMOVE + i); + } + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s realistic build with %u documents:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); +#endif + +#if 1 + TimePoint bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { + nns.addDoc(i); + } + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s clean build with %u documents:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); + + bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.addDoc(EFFECTIVE_DOCS + i); + } + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.removeDoc(EFFECTIVE_DOCS + i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index add then remove %u docs: %.3f ms\n", + name, NUM_DOCS_REMOVE, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s remove-damaged build with %u documents:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); +#endif + +#if 0 + TimePoint bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { + nns.addDoc(i); + } + TimePoint aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s clean build with %u documents:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); + + bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { + nns.removeDoc(i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index removed %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef)); + + const uint32_t addFirst = NUM_DOCS - (NUM_DOCS_REMOVE * 3); + const uint32_t addSecond = NUM_DOCS - (NUM_DOCS_REMOVE * 2); + + bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < addFirst; ++i) { + nns.addDoc(i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, addFirst, to_ms(aft - bef)); + + bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.addDoc(EFFECTIVE_DOCS + i); + nns.addDoc(addFirst + i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index added %u docs: %.3f ms\n", + name, 2 * NUM_DOCS_REMOVE, to_ms(aft - bef)); + + bef = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) { + nns.removeDoc(EFFECTIVE_DOCS + i); + nns.addDoc(addSecond + i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index added %u and removed %u docs: %.3f ms\n", + name, NUM_DOCS_REMOVE, NUM_DOCS_REMOVE, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s with %u documents some churn:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); + +#endif + +#if 0 + bef = std::chrono::steady_clock::now(); + fprintf(stderr, "removing and adding %u documents...\n", EFFECTIVE_DOCS); + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { + nns.removeDoc(i); + nns.addDoc(i); + } + aft = std::chrono::steady_clock::now(); + fprintf(stderr, "build %s index rem/add %u docs: %.3f ms\n", + name, EFFECTIVE_DOCS, to_ms(aft - bef)); + + timing_nns(name, nns, sk_list); + fprintf(stderr, "Quality for %s with %u documents full churn:\n", name, EFFECTIVE_DOCS); + quality_nns(nns, sk_list); +#endif +} + +#if 0 +TEST("require that Locality Sensitive Hashing mostly works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_rplsh_nns(NUM_DIMS, adapter); + benchmark_nns("RPLSH", *nns, { 200, 1000 }); +} +#endif + +#if 0 +TEST("require that Annoy via NNS api mostly works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_annoy_nns(NUM_DIMS, adapter); + benchmark_nns("Annoy", *nns, { 8000, 10000 }); +} +#endif + +#if 1 +TEST("require that HNSW via NNS api mostly works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_hnsw_nns(NUM_DIMS, adapter); + benchmark_nns("HNSW-like", *nns, { 100, 150, 200 }); +} +#endif + +#if 0 +TEST("require that HNSW wrapped api mostly works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_hnsw_wrap(NUM_DIMS, adapter); + benchmark_nns("HNSW-wrap", *nns, { 100, 150, 200 }); +} +#endif + +/** + * Before running the benchmark the ANN_GIST1M data set must be downloaded and extracted: + * wget ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz + * tar -xf gist.tar.gz + * + * The benchmark program will load the data set from $HOME/gist if no directory is specified. + * + * More information about the dataset is found here: http://corpus-texmex.irisa.fr/. + */ +int main(int argc, char **argv) { + TEST_MASTER.init(__FILE__); + std::string gist_dir = "."; + if (argc > 1) { + gist_dir = argv[1]; + } else { + char *home = getenv("HOME"); + if (home) { + gist_dir = home; + gist_dir += "/gist"; + } + } + read_data(gist_dir); + TEST_RUN_ALL(); + return (TEST_MASTER.fini() ? 0 : 1); +} diff --git a/eval/src/tests/ann/sift_benchmark.cpp b/eval/src/tests/ann/sift_benchmark.cpp index f20df926f24..022c9404f5d 100644 --- a/eval/src/tests/ann/sift_benchmark.cpp +++ b/eval/src/tests/ann/sift_benchmark.cpp @@ -8,6 +8,7 @@ #include <unistd.h> #include <stdio.h> #include <chrono> +#include <cstdlib> #define NUM_DIMS 128 #define NUM_DOCS 1000000 @@ -28,12 +29,15 @@ struct PointVector { }; static PointVector *aligned_alloc(size_t num) { - char *mem = (char *)malloc(num * sizeof(PointVector) + 512); + size_t sz = num * sizeof(PointVector); + double mega_bytes = sz / (1024.0*1024.0); + fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes); + char *mem = (char *)malloc(sz + 512); mem += 512; size_t val = (size_t)mem; size_t unalign = val % 512; mem -= unalign; - return (PointVector *)mem; + return reinterpret_cast<PointVector *>(mem); } static PointVector *generatedQueries = aligned_alloc(NUM_Q); @@ -92,13 +96,14 @@ double to_ms(Duration elapsed) { return ms.count(); } -void read_data(std::string dir) { +void read_data(const std::string& dir, const std::string& data_set) { + fprintf(stderr, "read data set '%s' from directory '%s'\n", data_set.c_str(), dir.c_str()); TimePoint bef = std::chrono::steady_clock::now(); - read_queries(dir + "/sift_query.fvecs"); + read_queries(dir + "/" + data_set + "_query.fvecs"); TimePoint aft = std::chrono::steady_clock::now(); fprintf(stderr, "read queries: %.3f ms\n", to_ms(aft - bef)); bef = std::chrono::steady_clock::now(); - read_docs(dir + "/sift_base.fvecs"); + read_docs(dir + "/" + data_set + "_base.fvecs"); aft = std::chrono::steady_clock::now(); fprintf(stderr, "read docs: %.3f ms\n", to_ms(aft - bef)); } @@ -168,7 +173,7 @@ void verifyBF(uint32_t qid) { fprintf(stderr, "WARN dist %.9g < mindist %.9g\n", dist, min_distance); } EXPECT_FALSE(dist+0.000001 < min_distance); - if (qid == 6) all_c2.push_back(dist / min_distance); + if (min_distance > 0) all_c2.push_back(dist / min_distance); } if (all_c2.size() != NUM_DOCS) return; std::sort(all_c2.begin(), all_c2.end()); @@ -280,24 +285,56 @@ TEST("require that Annoy via NNS api mostly works") { TEST("require that HNSW via NNS api mostly works") { DocVectorAdapter adapter; std::unique_ptr<NNS_API> nns = make_hnsw_nns(NUM_DIMS, adapter); - benchmark_nns("HNSW", *nns, { 100, 200 }); + benchmark_nns("HNSW-like", *nns, { 100, 150, 200 }); } #endif +#if 0 +TEST("require that HNSW wrapped api mostly works") { + DocVectorAdapter adapter; + std::unique_ptr<NNS_API> nns = make_hnsw_wrap(NUM_DIMS, adapter); + benchmark_nns("HNSW-wrap", *nns, { 100, 150, 200 }); +} +#endif +/** + * Before running the benchmark the ANN_SIFT1M data set must be downloaded and extracted: + * wget ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz + * tar -xf sift.tar.gz + * + * To run the program: + * ./eval_sift_benchmark_app <data_dir> + * + * The benchmark program will load the data set from $HOME/sift if no directory is specified. + * + * + * The ANN_GIST1M data set can also be used (as it has the same file format): + * wget ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz + * tar -xf gist.tar.gz + * + * Note that #define NUM_DIMS must be changed to 960 before recompiling and running the program: + * ./eval_sift_benchmark_app gist <data_dir> + * + * + * More information about the datasets is found here: http://corpus-texmex.irisa.fr/. + */ int main(int argc, char **argv) { TEST_MASTER.init(__FILE__); - std::string sift_dir = "."; - if (argc > 1) { - sift_dir = argv[1]; + std::string data_set = "sift"; + std::string data_dir = "."; + if (argc > 2) { + data_set = argv[1]; + data_dir = argv[2]; + } else if (argc > 1) { + data_dir = argv[1]; } else { char *home = getenv("HOME"); if (home) { - sift_dir = home; - sift_dir += "/sift"; + data_dir = home; + data_dir += "/" + data_set; } } - read_data(sift_dir); + read_data(data_dir, data_set); TEST_RUN_ALL(); return (TEST_MASTER.fini() ? 0 : 1); } diff --git a/eval/src/tests/ann/xp-annoy-nns.cpp b/eval/src/tests/ann/xp-annoy-nns.cpp index 45392084c80..f022aae5974 100644 --- a/eval/src/tests/ann/xp-annoy-nns.cpp +++ b/eval/src/tests/ann/xp-annoy-nns.cpp @@ -3,6 +3,7 @@ #include "nns.h" #include "std-random.h" #include <assert.h> +#include <cinttypes> #include <algorithm> #include <queue> #include <set> @@ -11,11 +12,11 @@ using V = vespalib::ConstArrayRef<float>; class AnnoyLikeNns; struct Node; -static uint64_t plane_dist_cnt = 0; -static uint64_t w_cen_dist_cnt = 0; -static uint64_t leaf_split_cnt = 0; -static uint64_t find_top_k_cnt = 0; -static uint64_t find_cand_cnt = 0; +static size_t plane_dist_cnt = 0; +static size_t w_cen_dist_cnt = 0; +static size_t leaf_split_cnt = 0; +static size_t find_top_k_cnt = 0; +static size_t find_cand_cnt = 0; using QueueNode = std::pair<double, Node *>; using NodeQueue = std::priority_queue<QueueNode>; diff --git a/eval/src/tests/ann/xp-hnsw-wrap.cpp b/eval/src/tests/ann/xp-hnsw-wrap.cpp new file mode 100644 index 00000000000..3eb01142dcd --- /dev/null +++ b/eval/src/tests/ann/xp-hnsw-wrap.cpp @@ -0,0 +1,56 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nns.h" +#include <iostream> +#include "/git/hnswlib/hnswlib/hnswlib.h" + +class HnswWrapNns : public NNS<float> +{ +private: + using Implementation = hnswlib::HierarchicalNSW<float>; + hnswlib::L2Space _l2space; + Implementation _hnsw; + +public: + HnswWrapNns(uint32_t numDims, const DocVectorAccess<float> &dva) + : NNS(numDims, dva), + _l2space(numDims), + _hnsw(&_l2space, 2500000, 16, 200) + { + } + + ~HnswWrapNns() {} + + void addDoc(uint32_t docid) override { + Vector vector = _dva.get(docid); + _hnsw.addPoint(vector.cbegin(), docid); + } + + void removeDoc(uint32_t docid) override { + _hnsw.markDelete(docid); + } + + std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override { + std::vector<NnsHit> reversed; + _hnsw.setEf(search_k); + auto priQ = _hnsw.searchKnn(vector.cbegin(), k); + while (! priQ.empty()) { + auto pair = priQ.top(); + reversed.emplace_back(pair.second, SqDist(pair.first)); + priQ.pop(); + } + std::vector<NnsHit> result; + while (result.size() < k && !reversed.empty()) { + result.push_back(reversed.back()); + reversed.pop_back(); + } + return result; + } +}; + +std::unique_ptr<NNS<float>> +make_hnsw_wrap(uint32_t numDims, const DocVectorAccess<float> &dva) +{ + NNS<float> *p = new HnswWrapNns(numDims, dva); + return std::unique_ptr<NNS<float>>(p); +} diff --git a/eval/src/tests/ann/xp-hnswlike-nns.cpp b/eval/src/tests/ann/xp-hnswlike-nns.cpp index ec831610a71..5cdbdd8efa3 100644 --- a/eval/src/tests/ann/xp-hnswlike-nns.cpp +++ b/eval/src/tests/ann/xp-hnswlike-nns.cpp @@ -3,10 +3,55 @@ #include <algorithm> #include <assert.h> #include <queue> -#include <random> +#include <cinttypes> +#include "std-random.h" #include "nns.h" -using LinkList = std::vector<uint32_t>; +/* + Todo: + + measure effect of: + 1) removing leftover backlinks during "shrink" operation + 2) refilling to low-watermark after 1) happens + 3) refilling to mid-watermark after 1) happens + 4) adding then removing 20% extra documents + 5) removing 20% first-added documents + 6) removing first-added documents while inserting new ones + + 7) auto-tune search_k to ensure >= 50% recall on 1000 Q with k=100 + 8) auto-tune search_k to ensure avg 90% recall on 1000 Q with k=100 + 9) auto-tune search_k to ensure >= 90% reachability of 10000 docids + + 10) timings for SIFT, GIST, and DEEP data (100k, 200k, 300k, 500k, 700k, 1000k) + */ + +static size_t distcalls_simple; +static size_t distcalls_search_layer; +static size_t distcalls_other; +static size_t distcalls_heuristic; +static size_t distcalls_shrink; +static size_t distcalls_refill; +static size_t refill_needed_calls; + +struct LinkList : std::vector<uint32_t> +{ + bool has_link_to(uint32_t id) const { + auto iter = std::find(begin(), end(), id); + return (iter != end()); + } + void remove_link(uint32_t id) { + uint32_t last = back(); + for (iterator iter = begin(); iter != end(); ++iter) { + if (*iter == id) { + *iter = last; + pop_back(); + return; + } + } + fprintf(stderr, "BAD missing link to remove: %u\n", id); + abort(); + } +}; struct Node { std::vector<LinkList> _links; @@ -61,38 +106,36 @@ struct VisitedSetPool }; struct HnswHit { - float dist; + double dist; uint32_t docid; HnswHit(uint32_t di, SqDist sq) : dist(sq.distance), docid(di) {} }; - -using QueueEntry = HnswHit; struct GreaterDist { - bool operator() (const QueueEntry &lhs, const QueueEntry& rhs) const { + bool operator() (const HnswHit &lhs, const HnswHit& rhs) const { return (rhs.dist < lhs.dist); } }; struct LesserDist { - bool operator() (const QueueEntry &lhs, const QueueEntry& rhs) const { + bool operator() (const HnswHit &lhs, const HnswHit& rhs) const { return (lhs.dist < rhs.dist); } }; -using NearestList = std::vector<QueueEntry>; +using NearestList = std::vector<HnswHit>; -struct NearestPriQ : std::priority_queue<QueueEntry, NearestList, GreaterDist> +struct NearestPriQ : std::priority_queue<HnswHit, NearestList, GreaterDist> { }; -struct FurthestPriQ : std::priority_queue<QueueEntry, NearestList, LesserDist> +struct FurthestPriQ : std::priority_queue<HnswHit, NearestList, LesserDist> { - NearestList steal() { - NearestList result; - c.swap(result); - return result; - } - const NearestList& peek() const { return c; } + NearestList steal() { + NearestList result; + c.swap(result); + return result; + } + const NearestList& peek() const { return c; } }; class HnswLikeNns : public NNS<float> @@ -104,8 +147,9 @@ private: uint32_t _M; uint32_t _efConstruction; double _levelMultiplier; - std::default_random_engine _rndGen; + RndGen _rndGen; VisitedSetPool _visitedSetPool; + size_t _ops_counter; double distance(Vector v, uint32_t id) const; @@ -115,11 +159,14 @@ private: } int randomLevel() { - std::uniform_real_distribution<double> distribution(0.0, 1.0); - double r = -log(distribution(_rndGen)) * _levelMultiplier; + double unif = _rndGen.nextUniform(); + double r = -log(1.0-unif) * _levelMultiplier; return (int) r; } + uint32_t count_reachable() const; + void dumpStats() const; + public: HnswLikeNns(uint32_t numDims, const DocVectorAccess<float> &dva) : NNS(numDims, dva), @@ -127,13 +174,14 @@ public: _entryId(0), _entryLevel(-1), _M(16), - _efConstruction(150), - _levelMultiplier(1.0 / log(1.0 * _M)) + _efConstruction(200), + _levelMultiplier(1.0 / log(1.0 * _M)), + _rndGen(), + _ops_counter(0) { - _nodes.reserve(1234567); } - ~HnswLikeNns() {} + ~HnswLikeNns() { dumpStats(); } LinkList& getLinkList(uint32_t docid, uint32_t level) { // assert(docid < _nodes.size()); @@ -141,85 +189,43 @@ public: return _nodes[docid]._links[level]; } + const LinkList& getLinkList(uint32_t docid, uint32_t level) const { + return _nodes[docid]._links[level]; + } + // simple greedy search - QueueEntry search_layer_simple(Vector vector, QueueEntry curPoint, uint32_t searchLevel) { + HnswHit search_layer_simple(Vector vector, HnswHit curPoint, uint32_t searchLevel) { bool keepGoing = true; while (keepGoing) { keepGoing = false; const LinkList& neighbors = getLinkList(curPoint.docid, searchLevel); for (uint32_t n_id : neighbors) { double dist = distance(vector, n_id); + ++distcalls_simple; if (dist < curPoint.dist) { - curPoint = QueueEntry(n_id, SqDist(dist)); - keepGoing = true; + curPoint = HnswHit(n_id, SqDist(dist)); + keepGoing = true; } } } return curPoint; } - void search_layer_foradd(Vector vector, FurthestPriQ &w, - uint32_t ef, uint32_t searchLevel); - - FurthestPriQ search_layer(Vector vector, NearestList entryPoints, - uint32_t ef, uint32_t searchLevel) { - VisitedSet &visited = _visitedSetPool.get(_nodes.size()); - NearestPriQ candidates; - FurthestPriQ w; - for (auto point : entryPoints) { - candidates.push(point); - w.push(point); - visited.mark(point.docid); - } - double limd = std::numeric_limits<double>::max(); - while (! candidates.empty()) { - QueueEntry cand = candidates.top(); - candidates.pop(); - if (cand.dist > limd) { - break; - } - for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) { - if (visited.isMarked(e_id)) continue; - visited.mark(e_id); - double e_dist = distance(vector, e_id); - if (e_dist < limd) { - candidates.emplace(e_id, SqDist(e_dist)); - w.emplace(e_id, SqDist(e_dist)); - if (w.size() > ef) { - w.pop(); - limd = w.top().dist; - } - } - } - } - return w; - } + void search_layer(Vector vector, FurthestPriQ &w, + uint32_t ef, uint32_t searchLevel); - bool haveCloserDistance(QueueEntry e, const LinkList &r) const { + bool haveCloserDistance(HnswHit e, const LinkList &r) const { for (uint32_t prevId : r) { double dist = distance(e.docid, prevId); + ++distcalls_heuristic; if (dist < e.dist) return true; } return false; } - LinkList select_neighbors(NearestPriQ &&w, uint32_t curMax) const; + LinkList select_neighbors(const NearestList &neighbors, uint32_t curMax) const; - LinkList select_neighbors(const NearestList &neighbors, uint32_t curMax) { - if (neighbors.size() <= curMax) { - LinkList result; - result.reserve(curMax+1); - for (const auto & entry : neighbors) { - result.push_back(entry.docid); - } - return result; - } - NearestPriQ w; - for (const QueueEntry & entry : neighbors) { - w.push(entry); - } - return select_neighbors(std::move(w), curMax); - } + LinkList remove_weakest(const NearestList &neighbors, uint32_t curMax, LinkList &removed) const; void addDoc(uint32_t docid) override { Vector vector = _dva.get(docid); @@ -232,11 +238,13 @@ public: if (_entryLevel < 0) { _entryId = docid; _entryLevel = level; + track_ops(); return; } int searchLevel = _entryLevel; double entryDist = distance(vector, _entryId); - QueueEntry entryPoint(_entryId, SqDist(entryDist)); + ++distcalls_other; + HnswHit entryPoint(_entryId, SqDist(entryDist)); while (searchLevel > level) { entryPoint = search_layer_simple(vector, entryPoint, searchLevel); --searchLevel; @@ -245,9 +253,8 @@ public: FurthestPriQ w; w.push(entryPoint); while (searchLevel >= 0) { - search_layer_foradd(vector, w, _efConstruction, searchLevel); - uint32_t maxLinks = (searchLevel > 0) ? _M : (2 * _M); - LinkList neighbors = select_neighbors(w.peek(), maxLinks); + search_layer(vector, w, _efConstruction, searchLevel); + LinkList neighbors = select_neighbors(w.peek(), _M); connect_new_node(docid, neighbors, searchLevel); each_shrink_ifneeded(neighbors, searchLevel); --searchLevel; @@ -256,46 +263,121 @@ public: _entryLevel = level; _entryId = docid; } + track_ops(); + } + + void track_ops() { + _ops_counter++; + if ((_ops_counter % 10000) == 0) { + double div = _ops_counter; + fprintf(stderr, "add / remove ops: %zu\n", _ops_counter); + fprintf(stderr, "distance calls for layer: %zu is %.3f per op\n", distcalls_search_layer, distcalls_search_layer/ div); + fprintf(stderr, "distance calls for heuristic: %zu is %.3f per op\n", distcalls_heuristic, distcalls_heuristic / div); + fprintf(stderr, "distance calls for simple: %zu is %.3f per op\n", distcalls_simple, distcalls_simple / div); + fprintf(stderr, "distance calls for shrink: %zu is %.3f per op\n", distcalls_shrink, distcalls_shrink / div); + fprintf(stderr, "distance calls for refill: %zu is %.3f per op\n", distcalls_refill, distcalls_refill / div); + fprintf(stderr, "distance calls for other: %zu is %.3f per op\n", distcalls_other, distcalls_other / div); + fprintf(stderr, "refill needed calls: %zu is %.3f per op\n", refill_needed_calls, refill_needed_calls / div); + } + } + + void remove_link_from(uint32_t from_id, uint32_t remove_id, uint32_t level) { + LinkList &links = getLinkList(from_id, level); + links.remove_link(remove_id); + } + + void refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level) { + LinkList &my_links = getLinkList(my_id, level); + if (my_links.size() < 8) { + ++refill_needed_calls; + for (uint32_t repl_id : replacements) { + if (repl_id == my_id) continue; + if (my_links.has_link_to(repl_id)) continue; + LinkList &other_links = getLinkList(repl_id, level); + if (other_links.size() + 1 >= _M) continue; + other_links.push_back(my_id); + my_links.push_back(repl_id); + if (my_links.size() >= _M) return; + } + } } void connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level); + void shrink_links(uint32_t shrink_id, uint32_t maxLinks, uint32_t level) { + LinkList &links = getLinkList(shrink_id, level); + NearestList distances; + for (uint32_t n_id : links) { + double n_dist = distance(shrink_id, n_id); + ++distcalls_shrink; + distances.emplace_back(n_id, SqDist(n_dist)); + } + LinkList lostLinks; + LinkList oldLinks = links; + links = remove_weakest(distances, maxLinks, lostLinks); + for (uint32_t lost_id : lostLinks) { + remove_link_from(lost_id, shrink_id, level); + refill_ifneeded(lost_id, oldLinks, level); + } + } + void each_shrink_ifneeded(const LinkList &neighbors, uint32_t level); - void removeDoc(uint32_t ) override { + void removeDoc(uint32_t docid) override { + Node &node = _nodes[docid]; + bool need_new_entrypoint = (docid == _entryId); + for (int level = node._links.size(); level-- > 0; ) { + LinkList my_links; + my_links.swap(node._links[level]); + for (uint32_t n_id : my_links) { + if (need_new_entrypoint) { + _entryId = n_id; + _entryLevel = level; + need_new_entrypoint = false; + } + remove_link_from(n_id, docid, level); + } + for (uint32_t n_id : my_links) { + refill_ifneeded(n_id, my_links, level); + } + } + node = Node(docid, 0, _M); + if (need_new_entrypoint) { + _entryLevel = -1; + _entryId = 0; + for (uint32_t i = 0; i < _nodes.size(); ++i) { + if (_nodes[i]._links.size() > 0) { + _entryId = i; + _entryLevel = _nodes[i]._links.size() - 1; + break; + } + } + } + track_ops(); } std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override { std::vector<NnsHit> result; if (_entryLevel < 0) return result; double entryDist = distance(vector, _entryId); - QueueEntry entryPoint(_entryId, SqDist(entryDist)); + ++distcalls_other; + HnswHit entryPoint(_entryId, SqDist(entryDist)); int searchLevel = _entryLevel; + FurthestPriQ w; + w.push(entryPoint); while (searchLevel > 0) { - entryPoint = search_layer_simple(vector, entryPoint, searchLevel); + search_layer(vector, w, std::min(k, search_k), searchLevel); --searchLevel; } - NearestList entryPoints; - entryPoints.push_back(entryPoint); - FurthestPriQ w = search_layer(vector, entryPoints, std::max(k, search_k), 0); - if (w.size() < k) { - fprintf(stderr, "fewer than expected hits: k=%u, ks=%u, got=%zu\n", - k, search_k, w.size()); - } + search_layer(vector, w, std::max(k, search_k), 0); while (w.size() > k) { w.pop(); } - std::vector<QueueEntry> reversed; - reversed.reserve(w.size()); - while (! w.empty()) { - reversed.push_back(w.top()); - w.pop(); - } - result.reserve(reversed.size()); - while (! reversed.empty()) { - const QueueEntry &hit = reversed.back(); + NearestList tmp = w.steal(); + std::sort(tmp.begin(), tmp.end(), LesserDist()); + result.reserve(tmp.size()); + for (const auto & hit : tmp) { result.emplace_back(hit.docid, SqDist(hit.dist)); - reversed.pop_back(); } return result; } @@ -310,85 +392,219 @@ HnswLikeNns::distance(Vector v, uint32_t b) const void HnswLikeNns::each_shrink_ifneeded(const LinkList &neighbors, uint32_t level) { - uint32_t maxLinks = (level > 0) ? _M : (2 * _M); - for (uint32_t old_id : neighbors) { - LinkList &oldLinks = getLinkList(old_id, level); - if (oldLinks.size() > maxLinks) { - NearestPriQ w; - for (uint32_t n_id : oldLinks) { - double n_dist = distance(old_id, n_id); - w.emplace(n_id, SqDist(n_dist)); - } - oldLinks = select_neighbors(std::move(w), maxLinks); - } + uint32_t maxLinks = (level > 0) ? _M : (2 * _M); + for (uint32_t old_id : neighbors) { + LinkList &oldLinks = getLinkList(old_id, level); + if (oldLinks.size() > maxLinks) { + shrink_links(old_id, maxLinks, level); } + } } void -HnswLikeNns::search_layer_foradd(Vector vector, FurthestPriQ &w, - uint32_t ef, uint32_t searchLevel) +HnswLikeNns::search_layer(Vector vector, FurthestPriQ &w, + uint32_t ef, uint32_t searchLevel) { - NearestPriQ candidates; - VisitedSet &visited = _visitedSetPool.get(_nodes.size()); + NearestPriQ candidates; + VisitedSet &visited = _visitedSetPool.get(_nodes.size()); - for (const QueueEntry& entry : w.peek()) { - candidates.push(entry); - visited.mark(entry.docid); + for (const HnswHit & entry : w.peek()) { + candidates.push(entry); + visited.mark(entry.docid); + } + double limd = std::numeric_limits<double>::max(); + while (! candidates.empty()) { + HnswHit cand = candidates.top(); + if (cand.dist > limd) { + break; } - - double limd = std::numeric_limits<double>::max(); - while (! candidates.empty()) { - QueueEntry cand = candidates.top(); - candidates.pop(); - if (cand.dist > limd) { - break; - } - for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) { - if (visited.isMarked(e_id)) continue; - visited.mark(e_id); - double e_dist = distance(vector, e_id); - if (e_dist < limd) { - candidates.emplace(e_id, SqDist(e_dist)); - w.emplace(e_id, SqDist(e_dist)); - if (w.size() > ef) { - w.pop(); - limd = w.top().dist; - } + candidates.pop(); + for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) { + if (visited.isMarked(e_id)) continue; + visited.mark(e_id); + double e_dist = distance(vector, e_id); + ++distcalls_search_layer; + if (e_dist < limd) { + candidates.emplace(e_id, SqDist(e_dist)); + w.emplace(e_id, SqDist(e_dist)); + if (w.size() > ef) { + w.pop(); + limd = w.top().dist; } } } - return; + } + return; } LinkList -HnswLikeNns::select_neighbors(NearestPriQ &&w, uint32_t curMax) const { - LinkList result; - result.reserve(curMax+1); - while (! w.empty()) { - QueueEntry e = w.top(); - w.pop(); - if (haveCloserDistance(e, result)) continue; +HnswLikeNns::remove_weakest(const NearestList &neighbors, uint32_t curMax, LinkList &lost) const +{ + LinkList result; + result.reserve(curMax+1); + NearestPriQ w; + for (const auto & entry : neighbors) { + w.push(entry); + } + while (! w.empty()) { + HnswHit e = w.top(); + w.pop(); + if (result.size() == curMax || haveCloserDistance(e, result)) { + lost.push_back(e.docid); + } else { result.push_back(e.docid); - if (result.size() >= curMax) break; } - return result; + } + return result; } +#ifdef NO_BACKFILL +LinkList +HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const +{ + LinkList result; + result.reserve(curMax+1); + bool needFiltering = (neighbors.size() > curMax); + NearestPriQ w; + for (const auto & entry : neighbors) { + w.push(entry); + } + while (! w.empty()) { + HnswHit e = w.top(); + w.pop(); + if (needFiltering && haveCloserDistance(e, result)) { + continue; + } + result.push_back(e.docid); + if (result.size() == curMax) return result; + } + return result; +} +#else +LinkList +HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const +{ + LinkList result; + result.reserve(curMax+1); + bool needFiltering = (neighbors.size() > curMax); + NearestPriQ w; + for (const auto & entry : neighbors) { + w.push(entry); + } + LinkList backfill; + while (! w.empty()) { + HnswHit e = w.top(); + w.pop(); + if (needFiltering && haveCloserDistance(e, result)) { + backfill.push_back(e.docid); + continue; + } + result.push_back(e.docid); + if (result.size() == curMax) return result; + } + if (result.size() * 4 < curMax) { + for (uint32_t fill_id : backfill) { + result.push_back(fill_id); + if (result.size() * 4 >= curMax) break; + } + } + return result; +} +#endif + void HnswLikeNns::connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level) { - LinkList &newLinks = getLinkList(id, level); - for (uint32_t neigh_id : neighbors) { - LinkList &oldLinks = getLinkList(neigh_id, level); - newLinks.push_back(neigh_id); - oldLinks.push_back(id); + LinkList &newLinks = getLinkList(id, level); + for (uint32_t neigh_id : neighbors) { + LinkList &oldLinks = getLinkList(neigh_id, level); + newLinks.push_back(neigh_id); + oldLinks.push_back(id); + } +} + +uint32_t +HnswLikeNns::count_reachable() const { + VisitedSet visited(_nodes.size()); + int level = _entryLevel; + LinkList curList; + curList.push_back(_entryId); + visited.mark(_entryId); + uint32_t idx = 0; + while (level >= 0) { + while (idx < curList.size()) { + uint32_t id = curList[idx++]; + const LinkList &links = getLinkList(id, level); + for (uint32_t n_id : links) { + if (visited.isMarked(n_id)) continue; + visited.mark(n_id); + curList.push_back(n_id); + } } + --level; + idx = 0; + } + return curList.size(); } +void +HnswLikeNns::dumpStats() const { + std::vector<uint32_t> levelCounts; + levelCounts.resize(_entryLevel + 2); + std::vector<uint32_t> outLinkHist; + outLinkHist.resize(2 * _M + 2); + uint32_t symmetrics = 0; + uint32_t level1links = 0; + uint32_t both_l_links = 0; + fprintf(stderr, "stats for HnswLikeNns with %zu nodes, entry level = %d, entry id = %u\n", + _nodes.size(), _entryLevel, _entryId); + + for (uint32_t id = 0; id < _nodes.size(); ++id) { + const auto &node = _nodes[id]; + uint32_t levels = node._links.size(); + levelCounts[levels]++; + if (levels < 1) { + outLinkHist[0]++; + continue; + } + const LinkList &link_list = getLinkList(id, 0); + uint32_t numlinks = link_list.size(); + outLinkHist[numlinks]++; + if (numlinks < 1) { + fprintf(stderr, "node with %u links: id %u\n", numlinks, id); + } + bool all_sym = true; + for (uint32_t n_id : link_list) { + const LinkList &neigh_list = getLinkList(n_id, 0); + if (! neigh_list.has_link_to(id)) { + fprintf(stderr, "BAD: %u has link to neighbor %u, but backlink is missing\n", id, n_id); + all_sym = false; + } + } + if (all_sym) ++symmetrics; + if (levels < 2) continue; + const LinkList &link_list_1 = getLinkList(id, 1); + for (uint32_t n_id : link_list_1) { + ++level1links; + if (link_list.has_link_to(n_id)) ++both_l_links; + } + } + for (uint32_t l = 0; l < levelCounts.size(); ++l) { + fprintf(stderr, "Nodes on %u levels: %u\n", l, levelCounts[l]); + } + fprintf(stderr, "reachable nodes %u / %zu\n", + count_reachable(), _nodes.size() - levelCounts[0]); + fprintf(stderr, "level 1 links overlapping on l0: %u / total: %u\n", + both_l_links, level1links); + for (uint32_t l = 0; l < outLinkHist.size(); ++l) { + if (outLinkHist[l] != 0) { + fprintf(stderr, "Nodes with %u outward links on L0: %u\n", l, outLinkHist[l]); + } + } + fprintf(stderr, "Symmetric in-out nodes: %u\n", symmetrics); +} std::unique_ptr<NNS<float>> make_hnsw_nns(uint32_t numDims, const DocVectorAccess<float> &dva) { return std::make_unique<HnswLikeNns>(numDims, dva); } - - diff --git a/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt b/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt new file mode 100644 index 00000000000..7234e8b9e69 --- /dev/null +++ b/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_matmul_function_test_app TEST + SOURCES + dense_matmul_function_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_dense_matmul_function_test_app COMMAND eval_dense_matmul_function_test_app) diff --git a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp new file mode 100644 index 00000000000..5d7c0be704e --- /dev/null +++ b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp @@ -0,0 +1,147 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_matmul_function.h> +#include <vespa/eval/tensor/dense/dense_tensor.h> +#include <vespa/eval/tensor/dense/dense_tensor_view.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/eval_fixture.h> + +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/stash.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::tensor; +using namespace vespalib::eval::tensor_function; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +void add_matrix(EvalFixture::ParamRepo &repo, const char *d1, size_t s1, const char *d2, size_t s2) { + for (bool float_cells: {false, true}) { + auto name = make_string("%s%zu%s%zu%s", d1, s1, d2, s2, float_cells ? "f" : ""); + auto type_str = make_string("tensor%s(%s[%zu],%s[%zu])", float_cells ? "<float>" : "", d1, s1, d2, s2); + TensorSpec matrix(type_str); + for (size_t i = 0; i < s1; ++i) { + for (size_t j = 0; j < s2; ++j) { + double value = (i + s1 + s2) * 3.0 + (j + s2) * 7.0; + matrix.add({{d1, i}, {d2, j}}, value); + } + } + repo.add(name, matrix); + } +} + +EvalFixture::ParamRepo make_params() { + EvalFixture::ParamRepo repo; + add_matrix(repo, "a", 2, "d", 3); // inner/inner + add_matrix(repo, "a", 2, "b", 5); // inner/outer + add_matrix(repo, "b", 5, "c", 2); // outer/outer + add_matrix(repo, "a", 2, "c", 3); // not matching + //----------------------------------------------- + add_matrix(repo, "b", 5, "d", 3); // fixed param + return repo; +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, + size_t lhs_size, size_t common_size, size_t rhs_size, + bool lhs_inner, bool rhs_inner) +{ + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseMatMulFunction>(); + ASSERT_EQUAL(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQUAL(info[0]->lhs_size(), lhs_size); + EXPECT_EQUAL(info[0]->common_size(), common_size); + EXPECT_EQUAL(info[0]->rhs_size(), rhs_size); + EXPECT_EQUAL(info[0]->lhs_common_inner(), lhs_inner); + EXPECT_EQUAL(info[0]->rhs_common_inner(), rhs_inner); +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseMatMulFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST("require that matmul can be optimized") { + TEST_DO(verify_optimized("reduce(a2d3*b5d3,sum,d)", 2, 3, 5, true, true)); +} + +TEST("require that matmul with lambda can be optimized") { + TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, true, true)); + TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, true, true)); +} + +TEST("require that expressions similar to matmul are not optimized") { + TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,a)")); + TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,b)")); + TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,prod,d)")); + TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x+y)),sum,d)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*x)),sum,d)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*y)),sum,d)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y*1)),sum,d)")); + TEST_DO(verify_not_optimized("reduce(a2c3*b5d3,sum,d)")); + TEST_DO(verify_not_optimized("reduce(a2c3*b5d3,sum,c)")); +} + +TEST("require that xw product can be debug dumped") { + EvalFixture fixture(prod_engine, "reduce(a2d3*b5d3,sum,d)", param_repo, true); + auto info = fixture.find_all<DenseMatMulFunction>(); + ASSERT_EQUAL(info.size(), 1u); + fprintf(stderr, "%s\n", info[0]->as_string().c_str()); +} + +vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common, + bool float_a, bool float_b) +{ + return make_string("reduce(%s%s*%s%s,sum,%s)", a.c_str(), float_a ? "f" : "", b.c_str(), float_b ? "f" : "", common.c_str()); +} + +void verify_optimized_multi(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common, + size_t lhs_size, size_t common_size, size_t rhs_size, + bool lhs_inner, bool rhs_inner) +{ + for (bool float_a: {false, true}) { + for (bool float_b: {false, true}) { + { + auto expr = make_expr(a, b, common, float_a, float_b); + TEST_STATE(expr.c_str()); + TEST_DO(verify_optimized(expr, lhs_size, common_size, rhs_size, lhs_inner, rhs_inner)); + } + { + auto expr = make_expr(b, a, common, float_b, float_a); + TEST_STATE(expr.c_str()); + TEST_DO(verify_optimized(expr, lhs_size, common_size, rhs_size, lhs_inner, rhs_inner)); + } + } + } +} + +TEST("require that matmul inner/inner works correctly") { + TEST_DO(verify_optimized_multi("a2d3", "b5d3", "d", 2, 3, 5, true, true)); +} + +TEST("require that matmul inner/outer works correctly") { + TEST_DO(verify_optimized_multi("a2b5", "b5d3", "b", 2, 5, 3, true, false)); +} + +TEST("require that matmul outer/outer works correctly") { + TEST_DO(verify_optimized_multi("b5c2", "b5d3", "b", 2, 5, 3, false, false)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 426281686d7..0b924451907 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("dense_dot_product_function_test"); - #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/operation.h> @@ -26,6 +23,12 @@ using namespace vespalib::eval::tensor_function; const TensorEngine &prod_engine = DefaultTensorEngine::ref(); +struct First { + bool value; + explicit First(bool value_in) : value(value_in) {} + operator bool() const { return value; } +}; + struct MyVecSeq : Sequence { double operator[](size_t i) const override { return (3.0 + i) * 7.0; } }; @@ -34,65 +37,100 @@ struct MyMatSeq : Sequence { double operator[](size_t i) const override { return (5.0 + i) * 43.0; } }; +void add_vector(EvalFixture::ParamRepo &repo, const char *d1, size_t s1) { + auto name = make_string("%s%zu", d1, s1); + auto layout = Layout({{d1, s1}}); + repo.add(name, spec(layout, MyVecSeq())); + repo.add(name + "f", spec(float_cells(layout), MyVecSeq())); +} + +void add_matrix(EvalFixture::ParamRepo &repo, const char *d1, size_t s1, const char *d2, size_t s2) { + auto name = make_string("%s%zu%s%zu", d1, s1, d2, s2); + auto layout = Layout({{d1, s1}, {d2, s2}}); + repo.add(name, spec(layout, MyMatSeq())); + repo.add(name + "f", spec(float_cells(layout), MyMatSeq())); +} + EvalFixture::ParamRepo make_params() { - return EvalFixture::ParamRepo() - .add("y1", spec({y(1)}, MyVecSeq())) - .add("y3", spec({y(3)}, MyVecSeq())) - .add("y3f", spec(float_cells({y(3)}), MyVecSeq())) - .add("y5", spec({y(5)}, MyVecSeq())) - .add("y16", spec({y(16)}, MyVecSeq())) - .add("x1y1", spec({x(1),y(1)}, MyMatSeq())) - .add("y1z1", spec({y(1),z(1)}, MyMatSeq())) - .add("x2y3", spec({x(2),y(3)}, MyMatSeq())) - .add("x2y3f", spec(float_cells({x(2),y(3)}), MyMatSeq())) - .add("y3z2f", spec(float_cells({y(3),z(2)}), MyMatSeq())) - .add("x2z3", spec({x(2),z(3)}, MyMatSeq())) - .add("y3z2", spec({y(3),z(2)}, MyMatSeq())) - .add("x8y5", spec({x(8),y(5)}, MyMatSeq())) - .add("y5z8", spec({y(5),z(8)}, MyMatSeq())) - .add("x5y16", spec({x(5),y(16)}, MyMatSeq())) - .add("y16z5", spec({y(16),z(5)}, MyMatSeq())); + EvalFixture::ParamRepo repo; + add_vector(repo, "y", 1); + add_vector(repo, "y", 3); + add_vector(repo, "y", 5); + add_vector(repo, "y", 16); + add_matrix(repo, "x", 1, "y", 1); + add_matrix(repo, "y", 1, "z", 1); + add_matrix(repo, "x", 2, "y", 3); + add_matrix(repo, "y", 3, "z", 2); + add_matrix(repo, "x", 2, "z", 3); + add_matrix(repo, "x", 8, "y", 5); + add_matrix(repo, "y", 5, "z", 8); + add_matrix(repo, "x", 5, "y", 16); + add_matrix(repo, "y", 16, "z", 5); + return repo; } EvalFixture::ParamRepo param_repo = make_params(); void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_size, bool happy) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); EvalFixture fixture(prod_engine, expr, param_repo, true); EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); auto info = fixture.find_all<DenseXWProductFunction>(); ASSERT_EQUAL(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); - EXPECT_EQUAL(info[0]->vectorSize(), vec_size); - EXPECT_EQUAL(info[0]->resultSize(), res_size); - EXPECT_EQUAL(info[0]->matrixHasCommonDimensionInnermost(), happy); + EXPECT_EQUAL(info[0]->vector_size(), vec_size); + EXPECT_EQUAL(info[0]->result_size(), res_size); + EXPECT_EQUAL(info[0]->common_inner(), happy); +} + +vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common, + bool float_a, bool float_b) +{ + return make_string("reduce(%s%s*%s%s,sum,%s)", a.c_str(), float_a ? "f" : "", b.c_str(), float_b ? "f" : "", common.c_str()); +} + +void verify_optimized_multi(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common, + size_t vec_size, size_t res_size, bool happy, First first = First(true)) +{ + for (bool float_a: {false, true}) { + for (bool float_b: {false, true}) { + auto expr = make_expr(a, b, common, float_a, float_b); + TEST_STATE(expr.c_str()); + TEST_DO(verify_optimized(expr, vec_size, res_size, happy)); + } + } + if (first) { + TEST_DO(verify_optimized_multi(b, a, common, vec_size, res_size, happy, First(false))); + } } void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); EvalFixture fixture(prod_engine, expr, param_repo, true); EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); auto info = fixture.find_all<DenseXWProductFunction>(); EXPECT_TRUE(info.empty()); } TEST("require that xw product gives same results as reference join/reduce") { // 1 -> 1 happy/unhappy - TEST_DO(verify_optimized("reduce(y1*x1y1,sum,y)", 1, 1, true)); - TEST_DO(verify_optimized("reduce(y1*y1z1,sum,y)", 1, 1, false)); + TEST_DO(verify_optimized_multi("y1", "x1y1", "y", 1, 1, true)); + TEST_DO(verify_optimized_multi("y1", "y1z1", "y", 1, 1, false)); // 3 -> 2 happy/unhappy - TEST_DO(verify_optimized("reduce(y3*x2y3,sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(y3*y3z2,sum,y)", 3, 2, false)); + TEST_DO(verify_optimized_multi("y3", "x2y3", "y", 3, 2, true)); + TEST_DO(verify_optimized_multi("y3", "y3z2", "y", 3, 2, false)); // 5 -> 8 happy/unhappy - TEST_DO(verify_optimized("reduce(y5*x8y5,sum,y)", 5, 8, true)); - TEST_DO(verify_optimized("reduce(y5*y5z8,sum,y)", 5, 8, false)); + TEST_DO(verify_optimized_multi("y5", "x8y5", "y", 5, 8, true)); + TEST_DO(verify_optimized_multi("y5", "y5z8", "y", 5, 8, false)); // 16 -> 5 happy/unhappy - TEST_DO(verify_optimized("reduce(y16*x5y16,sum,y)", 16, 5, true)); - TEST_DO(verify_optimized("reduce(y16*y16z5,sum,y)", 16, 5, false)); + TEST_DO(verify_optimized_multi("y16", "x5y16", "y", 16, 5, true)); + TEST_DO(verify_optimized_multi("y16", "y16z5", "y", 16, 5, false)); } TEST("require that various variants of xw product can be optimized") { - TEST_DO(verify_optimized("reduce(y3*x2y3,sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(x2y3*y3,sum,y)", 3, 2, true)); TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(x*y)),sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(join(x2y3,y3,f(x,y)(x*y)),sum,y)", 3, 2, true)); + TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)", 3, 2, true)); } TEST("require that expressions similar to xw product are not optimized") { @@ -100,13 +138,9 @@ TEST("require that expressions similar to xw product are not optimized") { TEST_DO(verify_not_optimized("reduce(y3*x2y3,prod,y)")); TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x+y)),sum,y)")); - // TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*y)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x*1)),sum,y)")); -} - -TEST("require that xw products with incompatible dimensions are not optimized") { TEST_DO(verify_not_optimized("reduce(y3*x2z3,sum,y)")); TEST_DO(verify_not_optimized("reduce(y3*x2z3,sum,z)")); } @@ -119,16 +153,4 @@ TEST("require that xw product can be debug dumped") { fprintf(stderr, "%s\n", info[0]->as_string().c_str()); } -TEST("require that optimization works for float cells") { - TEST_DO(verify_optimized("reduce(y3f*x2y3,sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(y3*x2y3f,sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(y3f*x2y3f,sum,y)", 3, 2, true)); -} - -TEST("require that optimization works for float cells with inconvenient dimension nesting") { - TEST_DO(verify_optimized("reduce(y3f*y3z2,sum,y)", 3, 2, false)); - TEST_DO(verify_optimized("reduce(y3*y3z2f,sum,y)", 3, 2, false)); - TEST_DO(verify_optimized("reduce(y3f*y3z2f,sum,y)", 3, 2, false)); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index 90972de7c80..c28643e605e 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -14,3 +14,6 @@ vespa_add_library(vespaeval DEPENDS ${VESPA_LLVM_LIB} ) + +set(BLA_VENDOR OpenBLAS) +vespa_add_target_package_dependency(vespaeval BLAS) diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index 65cec9c0d48..aaadec772a5 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -5,6 +5,7 @@ #include "compiled_function.h" #include <vespa/vespalib/util/executor.h> #include <condition_variable> +#include <atomic> #include <mutex> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 1d5515d7f4a..cce9838d967 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -12,15 +12,18 @@ #include <llvm/Analysis/Passes.h> #include <llvm/IR/DataLayout.h> #include <llvm/Transforms/Scalar.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) // Avoid reference to undefined symbol llvm::cfg::Update<llvm::BasicBlock*>::dump() const #define NDEBUG #endif #include <llvm/LinkAllPasses.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) #undef NDEBUG #endif #include <llvm/Transforms/IPO/PassManagerBuilder.h> +#if LLVM_VERSION_MAJOR > 9 +#include <llvm/Support/ManagedStatic.h> +#endif #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/approx.h> diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index e578b28da18..1e17c8284cb 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -83,6 +83,19 @@ std::vector<Value::CREF> get_refs(const std::vector<Value::UP> &values) { } // namespace vespalib::eval::test +void +EvalFixture::detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const +{ + for (size_t i = 0; i < _function->num_params(); ++i) { + auto pos = param_repo.map.find(_function->param_name(i)); + ASSERT_TRUE(pos != param_repo.map.end()); + bool allow_tampering = allow_mutable && pos->second.is_mutable; + if (!allow_tampering) { + ASSERT_EQUAL(pos->second.value, _engine.to_spec(*_param_values[i])); + } + } +} + EvalFixture::EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo ¶m_repo, @@ -104,6 +117,7 @@ EvalFixture::EvalFixture(const TensorEngine &engine, { auto result_type = ValueType::from_spec(_result.type()); ASSERT_TRUE(!result_type.is_error()); + TEST_DO(detect_param_tampering(param_repo, allow_mutable)); } const TensorSpec diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 48f6a7e5d2e..1d39dc52cba 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -67,6 +67,8 @@ private: } } + void detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const; + public: EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo ¶m_repo, bool optimized = true, bool allow_mutable = false); diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 1a4182dff2a..3e91240048b 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -46,9 +46,9 @@ private: : _type(type_in), _cell_type(cell_type_in), _dimensions(std::move(dimensions_in)) {} public: - ValueType(ValueType &&) = default; + ValueType(ValueType &&) noexcept = default; ValueType(const ValueType &) = default; - ValueType &operator=(ValueType &&) = default; + ValueType &operator=(ValueType &&) noexcept = default; ValueType &operator=(const ValueType &) = default; ~ValueType(); Type type() const { return _type; } diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index b4449309812..a9e1ad84eb7 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -10,6 +10,7 @@ #include "dense/typed_dense_tensor_builder.h" #include "dense/dense_dot_product_function.h" #include "dense/dense_xw_product_function.h" +#include "dense/dense_matmul_function.h" #include "dense/dense_fast_rename_optimizer.h" #include "dense/dense_add_dimension_optimizer.h" #include "dense/dense_remove_dimension_optimizer.h" @@ -269,6 +270,7 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const child.set(DenseTensorPeekFunction::optimize(child.get(), stash)); child.set(DenseDotProductFunction::optimize(child.get(), stash)); child.set(DenseXWProductFunction::optimize(child.get(), stash)); + child.set(DenseMatMulFunction::optimize(child.get(), stash)); child.set(DenseFastRenameOptimizer::optimize(child.get(), stash)); child.set(DenseAddDimensionOptimizer::optimize(child.get(), stash)); child.set(DenseRemoveDimensionOptimizer::optimize(child.get(), stash)); diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index 2ad54d48ab3..635a49cb4a9 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(eval_tensor_dense OBJECT dense_fast_rename_optimizer.cpp dense_inplace_join_function.cpp dense_inplace_map_function.cpp + dense_matmul_function.cpp dense_remove_dimension_optimizer.cpp dense_replace_type_function.cpp dense_tensor.cpp diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 8bcaddba3b4..f2c374ea02d 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -7,6 +7,8 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/tensor/tensor.h> +#include <cblas.h> + namespace vespalib::tensor { using eval::ValueType; @@ -19,32 +21,29 @@ using namespace eval::operation; namespace { template <typename LCT, typename RCT> -struct HWSupport { - static double call(hwaccelrated::IAccelrated *, const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs) { - double result = 0.0; - for (size_t i = 0; i < lhs.size(); ++i) { - result += (lhs[i] * rhs[i]); - } - return result; +void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + double result = 0.0; + const LCT *lhs = lhs_cells.cbegin(); + const RCT *rhs = rhs_cells.cbegin(); + for (size_t i = 0; i < lhs_cells.size(); ++i) { + result += ((*lhs++) * (*rhs++)); } -}; -template <> struct HWSupport<float, float> { - static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<float> &lhs, const ConstArrayRef<float> &rhs) { - return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size()); - } -}; -template <> struct HWSupport<double, double> { - static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<double> &lhs, const ConstArrayRef<double> &rhs) { - return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size()); - } -}; + state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); +} -template <typename LCT, typename RCT> -void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t param) { - auto *hw = (hwaccelrated::IAccelrated *)(param); - auto lhs = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto rhs = DenseTensorView::typify_cells<RCT>(state.peek(0)); - double result = HWSupport<LCT,RCT>::call(hw, lhs, rhs); +void my_cblas_double_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + double result = cblas_ddot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); + state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); +} + +void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + double result = cblas_sdot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); } @@ -53,21 +52,31 @@ struct MyDotProductOp { static auto get_fun() { return my_dot_product_op<LCT,RCT>; } }; +eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { + if (lct == rct) { + if (lct == ValueType::CellType::DOUBLE) { + return my_cblas_double_dot_product_op; + } + if (lct == ValueType::CellType::FLOAT) { + return my_cblas_float_dot_product_op; + } + } + return select_2<MyDotProductOp>(lct, rct); +} + } // namespace vespalib::tensor::<unnamed> DenseDotProductFunction::DenseDotProductFunction(const eval::TensorFunction &lhs_in, const eval::TensorFunction &rhs_in) - : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in), - _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator()) + : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in) { } eval::InterpretedFunction::Instruction DenseDotProductFunction::compile_self(Stash &) const { - auto op = select_2<MyDotProductOp>(lhs().result_type().cell_type(), - rhs().result_type().cell_type()); - return eval::InterpretedFunction::Instruction(op, (uint64_t)(_hwAccelerator.get())); + auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type()); + return eval::InterpretedFunction::Instruction(op); } bool diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h index d6181d33887..1d8f749689b 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/eval/eval/tensor_function.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace vespalib::tensor { @@ -13,7 +12,6 @@ namespace vespalib::tensor { class DenseDotProductFunction : public eval::tensor_function::Op2 { private: - hwaccelrated::IAccelrated::UP _hwAccelerator; using ValueType = eval::ValueType; public: DenseDotProductFunction(const eval::TensorFunction &lhs_in, diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp new file mode 100644 index 00000000000..272ceb319c8 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -0,0 +1,236 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_matmul_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/objects/objectvisitor.h> +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/tensor/tensor.h> +#include <assert.h> + +#include <cblas.h> + +namespace vespalib::tensor { + +using eval::ValueType; +using eval::TensorFunction; +using eval::as; +using eval::Aggr; +using namespace eval::tensor_function; +using namespace eval::operation; + +namespace { + +template <typename LCT, typename RCT, bool lhs_common_inner, bool rhs_common_inner> +double my_dot_product(const LCT *lhs, const RCT *rhs, size_t lhs_size, size_t common_size, size_t rhs_size) { + double result = 0.0; + for (size_t i = 0; i < common_size; ++i) { + result += ((*lhs) * (*rhs)); + lhs += (lhs_common_inner ? 1 : lhs_size); + rhs += (rhs_common_inner ? 1 : rhs_size); + } + return result; +} + +template <typename LCT, typename RCT, bool lhs_common_inner, bool rhs_common_inner> +void my_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); + using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type; + auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto dst_cells = state.stash.create_array<OCT>(self.lhs_size * self.rhs_size); + OCT *dst = dst_cells.begin(); + const LCT *lhs = lhs_cells.cbegin(); + for (size_t i = 0; i < self.lhs_size; ++i) { + const RCT *rhs = rhs_cells.cbegin(); + for (size_t j = 0; j < self.rhs_size; ++j) { + *dst++ = my_dot_product<LCT,RCT,lhs_common_inner,rhs_common_inner>(lhs, rhs, self.lhs_size, self.common_size, self.rhs_size); + rhs += (rhs_common_inner ? self.common_size : 1); + } + lhs += (lhs_common_inner ? self.common_size : 1); + } + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} + +template <bool lhs_common_inner, bool rhs_common_inner> +void my_cblas_double_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); + auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto dst_cells = state.stash.create_array<double>(self.lhs_size * self.rhs_size); + cblas_dgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans, + self.lhs_size, self.rhs_size, self.common_size, 1.0, + lhs_cells.cbegin(), lhs_common_inner ? self.common_size : self.lhs_size, + rhs_cells.cbegin(), rhs_common_inner ? self.common_size : self.rhs_size, + 0.0, dst_cells.begin(), self.rhs_size); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} + +template <bool lhs_common_inner, bool rhs_common_inner> +void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param)); + auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto dst_cells = state.stash.create_array<float>(self.lhs_size * self.rhs_size); + cblas_sgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans, + self.lhs_size, self.rhs_size, self.common_size, 1.0, + lhs_cells.cbegin(), lhs_common_inner ? self.common_size : self.lhs_size, + rhs_cells.cbegin(), rhs_common_inner ? self.common_size : self.rhs_size, + 0.0, dst_cells.begin(), self.rhs_size); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} + +template <bool lhs_common_inner, bool rhs_common_inner> +struct MyMatMulOp { + template <typename LCT, typename RCT> + static auto get_fun() { return my_matmul_op<LCT,RCT,lhs_common_inner,rhs_common_inner>; } +}; + +template <bool lhs_common_inner, bool rhs_common_inner> +eval::InterpretedFunction::op_function my_select3(CellType lct, CellType rct) +{ + if (lct == rct) { + if (lct == ValueType::CellType::DOUBLE) { + return my_cblas_double_matmul_op<lhs_common_inner,rhs_common_inner>; + } + if (lct == ValueType::CellType::FLOAT) { + return my_cblas_float_matmul_op<lhs_common_inner,rhs_common_inner>; + } + } + return select_2<MyMatMulOp<lhs_common_inner,rhs_common_inner>>(lct, rct); +} + +template <bool lhs_common_inner> +eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct, + bool rhs_common_inner) +{ + if (rhs_common_inner) { + return my_select3<lhs_common_inner,true>(lct, rct); + } else { + return my_select3<lhs_common_inner,false>(lct, rct); + } +} + +eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, + bool lhs_common_inner, bool rhs_common_inner) +{ + if (lhs_common_inner) { + return my_select2<true>(lct, rct, rhs_common_inner); + } else { + return my_select2<false>(lct, rct, rhs_common_inner); + } +} + +bool is_matrix(const ValueType &type) { + return (type.is_dense() && (type.dimensions().size() == 2)); +} + +bool is_matmul(const eval::ValueType &a, const eval::ValueType &b, + const vespalib::string &reduce_dim, const eval::ValueType &result_type) +{ + size_t npos = ValueType::Dimension::npos; + return (is_matrix(a) && is_matrix(b) && is_matrix(result_type) && + (a.dimension_index(reduce_dim) != npos) && + (b.dimension_index(reduce_dim) != npos)); +} + +const ValueType::Dimension &dim(const TensorFunction &expr, size_t idx) { + return expr.result_type().dimensions()[idx]; +} + +size_t inv(size_t idx) { return (1 - idx); } + +const TensorFunction &create_matmul(const TensorFunction &a, const TensorFunction &b, + const vespalib::string &reduce_dim, const ValueType &result_type, Stash &stash) { + size_t a_idx = a.result_type().dimension_index(reduce_dim); + size_t b_idx = b.result_type().dimension_index(reduce_dim); + assert(a_idx != ValueType::Dimension::npos); + assert(b_idx != ValueType::Dimension::npos); + assert(dim(a, a_idx).size == dim(b, b_idx).size); + bool a_common_inner = (a_idx == 1); + bool b_common_inner = (b_idx == 1); + size_t a_size = dim(a, inv(a_idx)).size; + size_t b_size = dim(b, inv(b_idx)).size; + size_t common_size = dim(a, a_idx).size; + bool a_is_lhs = (dim(a, inv(a_idx)).name < dim(b, inv(b_idx)).name); + if (a_is_lhs) { + return stash.create<DenseMatMulFunction>(result_type, a, b, a_size, common_size, b_size, a_common_inner, b_common_inner); + } else { + return stash.create<DenseMatMulFunction>(result_type, b, a, b_size, common_size, a_size, b_common_inner, a_common_inner); + } +} + +} // namespace vespalib::tensor::<unnamed> + +DenseMatMulFunction::Self::Self(const eval::ValueType &result_type_in, + size_t lhs_size_in, + size_t common_size_in, + size_t rhs_size_in) + : result_type(result_type_in), + lhs_size(lhs_size_in), + common_size(common_size_in), + rhs_size(rhs_size_in) +{ +} + +DenseMatMulFunction::Self::~Self() = default; + +DenseMatMulFunction::DenseMatMulFunction(const eval::ValueType &result_type, + const eval::TensorFunction &lhs_in, + const eval::TensorFunction &rhs_in, + size_t lhs_size, + size_t common_size, + size_t rhs_size, + bool lhs_common_inner, + bool rhs_common_inner) + : Super(result_type, lhs_in, rhs_in), + _lhs_size(lhs_size), + _common_size(common_size), + _rhs_size(rhs_size), + _lhs_common_inner(lhs_common_inner), + _rhs_common_inner(rhs_common_inner) +{ +} + +DenseMatMulFunction::~DenseMatMulFunction() = default; + +eval::InterpretedFunction::Instruction +DenseMatMulFunction::compile_self(Stash &stash) const +{ + Self &self = stash.create<Self>(result_type(), _lhs_size, _common_size, _rhs_size); + auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type(), + _lhs_common_inner, _rhs_common_inner); + return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); +} + +void +DenseMatMulFunction::visit_self(vespalib::ObjectVisitor &visitor) const +{ + Super::visit_self(visitor); + visitor.visitInt("lhs_size", _lhs_size); + visitor.visitInt("common_size", _common_size); + visitor.visitInt("rhs_size", _rhs_size); + visitor.visitBool("lhs_common_inner", _lhs_common_inner); + visitor.visitBool("rhs_common_inner", _rhs_common_inner); +} + +const TensorFunction & +DenseMatMulFunction::optimize(const eval::TensorFunction &expr, Stash &stash) +{ + auto reduce = as<Reduce>(expr); + if (reduce && (reduce->aggr() == Aggr::SUM) && (reduce->dimensions().size() == 1)) { + auto join = as<Join>(reduce->child()); + if (join && (join->function() == Mul::f)) { + const TensorFunction &a = join->lhs(); + const TensorFunction &b = join->rhs(); + if (is_matmul(a.result_type(), b.result_type(), reduce->dimensions()[0], expr.result_type())) { + return create_matmul(a, b, reduce->dimensions()[0], expr.result_type(), stash); + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h new file mode 100644 index 00000000000..f0b6d8b6c19 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h @@ -0,0 +1,58 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> +#include "dense_tensor_view.h" + +namespace vespalib::tensor { + +/** + * Tensor function for dense matrix multiplication. + **/ +class DenseMatMulFunction : public eval::tensor_function::Op2 +{ + using Super = eval::tensor_function::Op2; +public: + struct Self { + eval::ValueType result_type; + size_t lhs_size; + size_t common_size; + size_t rhs_size; + Self(const eval::ValueType &result_type_in, + size_t lhs_size_in, size_t common_size_in, size_t rhs_size_in); + ~Self(); + }; + +private: + size_t _lhs_size; + size_t _common_size; + size_t _rhs_size; + bool _lhs_common_inner; + bool _rhs_common_inner; + +public: + DenseMatMulFunction(const eval::ValueType &result_type, + const eval::TensorFunction &lhs_in, + const eval::TensorFunction &rhs_in, + size_t lhs_size, + size_t common_size, + size_t rhs_size, + bool lhs_common_inner, + bool rhs_common_inner); + ~DenseMatMulFunction(); + + bool result_is_mutable() const override { return true; } + + size_t lhs_size() const { return _lhs_size; } + size_t common_size() const { return _common_size; } + size_t rhs_size() const { return _rhs_size; } + bool lhs_common_inner() const { return _lhs_common_inner; } + bool rhs_common_inner() const { return _rhs_common_inner; } + + eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override; + void visit_self(vespalib::ObjectVisitor &visitor) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 4ed6758dfde..e450aa49284 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -301,7 +301,7 @@ DenseTensorView::join(join_fun_t function, const Tensor &arg) const return joinDenseTensors(*this, arg, "join", function); } if (function == eval::operation::Mul::f) { - return dense::generic_join(*this, arg, [](double a, double b) { return (a * b); }); + return dense::generic_join(*this, arg, [](auto a, auto b) { return (a * b); }); } if (function == eval::operation::Add::f) { return dense::generic_join(*this, arg, [](double a, double b) { return (a + b); }); @@ -323,7 +323,7 @@ DenseTensorView::reduce_all(join_fun_t op, const std::vector<vespalib::string> & return dense::reduce(*this, dims, [](double a, double b) { return (a * b);}); } if (op == eval::operation::Add::f) { - return dense::reduce(*this, dims, [](double a, double b) { return (a + b);}); + return dense::reduce(*this, dims, [](auto a, auto b) { return (a + b);}); } return dense::reduce(*this, dims, op); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index 2db5b4e8f92..7b03f45e9cc 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -10,6 +10,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <assert.h> +#include <cblas.h> + namespace vespalib::tensor { using eval::ValueType; @@ -21,76 +23,59 @@ using namespace eval::operation; namespace { -template <typename LCT, typename RCT> -struct HWSupport { - static double call(hwaccelrated::IAccelrated *, const LCT *lhs, const RCT *rhs, size_t len) { - double result = 0.0; - for (size_t i = 0; i < len; ++i) { - result += (lhs[i] * rhs[i]); - } - return result; +template <typename LCT, typename RCT, bool common_inner> +double my_dot_product(const LCT *lhs, const RCT *rhs, size_t vector_size, size_t result_size) { + double result = 0.0; + for (size_t i = 0; i < vector_size; ++i) { + result += ((*lhs) * (*rhs)); + ++lhs; + rhs += (common_inner ? 1 : result_size); } -}; -template <> struct HWSupport<float, float> { - static double call(hwaccelrated::IAccelrated *hw, const float *lhs, const float *rhs, size_t len) { - return hw->dotProduct(lhs, rhs, len); - } -}; -template <> struct HWSupport<double, double> { - static double call(hwaccelrated::IAccelrated *hw, const double *lhs, const double *rhs, size_t len) { - return hw->dotProduct(lhs, rhs, len); - } -}; - -template <typename LCT, typename RCT, typename OCT> -void multiDotProduct(const DenseXWProductFunction::Self &self, - const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result) -{ - OCT *out = result.begin(); - const RCT *matrixP = matrixCells.cbegin(); - const LCT * const vectorP = vectorCells.cbegin(); - for (size_t row = 0; row < self._resultSize; ++row) { - double cell = HWSupport<LCT,RCT>::call(self._hwAccelerator.get(), vectorP, matrixP, self._vectorSize); - *out++ = cell; - matrixP += self._vectorSize; - } - assert(out == result.end()); - assert(matrixP == matrixCells.cend()); -} - -template <typename LCT, typename RCT, typename OCT> -void transposedProduct(const DenseXWProductFunction::Self &self, - const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result) -{ - OCT *out = result.begin(); - const RCT * const matrixP = matrixCells.cbegin(); - const LCT * const vectorP = vectorCells.cbegin(); - for (size_t row = 0; row < self._resultSize; ++row) { - double cell = 0; - for (size_t col = 0; col < self._vectorSize; ++col) { - cell += matrixP[col*self._resultSize + row] * vectorP[col]; - } - *out++ = cell; - } - assert(out == result.end()); + return result; } -template <typename LCT, typename RCT, bool commonDimensionInnermost> +template <typename LCT, typename RCT, bool common_inner> void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { - DenseXWProductFunction::Self *self = (DenseXWProductFunction::Self *)(param); - + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type; - auto vectorCells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto matrixCells = DenseTensorView::typify_cells<RCT>(state.peek(0)); - auto outputCells = state.stash.create_array<OCT>(self->_resultSize); - - if (commonDimensionInnermost) { - multiDotProduct(*self, vectorCells, matrixCells, outputCells); - } else { - transposedProduct(*self, vectorCells, matrixCells, outputCells); + auto vector_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto dst_cells = state.stash.create_array<OCT>(self.result_size); + OCT *dst = dst_cells.begin(); + const RCT *matrix = matrix_cells.cbegin(); + for (size_t i = 0; i < self.result_size; ++i) { + *dst++ = my_dot_product<LCT,RCT,common_inner>(vector_cells.cbegin(), matrix, self.vector_size, self.result_size); + matrix += (common_inner ? self.vector_size : 1); } + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} + +template <bool common_inner> +void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); + auto vector_cells = DenseTensorView::typify_cells<double>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto dst_cells = state.stash.create_array<double>(self.result_size); + cblas_dgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, + common_inner ? self.result_size : self.vector_size, + common_inner ? self.vector_size : self.result_size, + 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1, + 0.0, dst_cells.begin(), 1); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} - state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, TypedCells(outputCells))); +template <bool common_inner> +void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); + auto vector_cells = DenseTensorView::typify_cells<float>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto dst_cells = state.stash.create_array<float>(self.result_size); + cblas_sgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, + common_inner ? self.result_size : self.vector_size, + common_inner ? self.vector_size : self.result_size, + 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1, + 0.0, dst_cells.begin(), 1); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } template <bool common_inner> @@ -99,11 +84,24 @@ struct MyXWProductOp { static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; } }; -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_innermost) { - if (common_innermost) { - return select_2<MyXWProductOp<true> >(lct, rct); +template <bool common_inner> +eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) { + if (lct == rct) { + if (lct == ValueType::CellType::DOUBLE) { + return my_cblas_double_xw_product_op<common_inner>; + } + if (lct == ValueType::CellType::FLOAT) { + return my_cblas_float_xw_product_op<common_inner>; + } + } + return select_2<MyXWProductOp<common_inner>>(lct, rct); +} + +eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) { + if (common_inner) { + return my_select2<true>(lct, rct); } else { - return select_2<MyXWProductOp<false> >(lct, rct); + return my_select2<false>(lct, rct); } } @@ -129,42 +127,43 @@ bool isDenseXWProduct(const ValueType &res, const ValueType &vec, const ValueTyp } const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFunction &vec, const TensorFunction &mat, Stash &stash) { - bool common_is_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1); + bool common_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1); return stash.create<DenseXWProductFunction>(res, vec, mat, vec.result_type().dimensions()[0].size, res.dimensions()[0].size, - common_is_inner); + common_inner); } } // namespace vespalib::tensor::<unnamed> -DenseXWProductFunction::Self::Self(const eval::ValueType &resultType, - size_t vectorSize, - size_t resultSize) - : _resultType(resultType), - _vectorSize(vectorSize), - _resultSize(resultSize), - _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator()) -{} +DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in, + size_t vector_size_in, size_t result_size_in) + : result_type(result_type_in), + vector_size(vector_size_in), + result_size(result_size_in) +{ +} +DenseXWProductFunction::Self::~Self() = default; -DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &resultType, +DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &result_type, const eval::TensorFunction &vector_in, const eval::TensorFunction &matrix_in, - size_t vectorSize, - size_t resultSize, - bool matrixHasCommonDimensionInnermost) - : eval::tensor_function::Op2(resultType, vector_in, matrix_in), - _vectorSize(vectorSize), - _resultSize(resultSize), - _commonDimensionInnermost(matrixHasCommonDimensionInnermost) -{} + size_t vector_size, + size_t result_size, + bool common_inner) + : eval::tensor_function::Op2(result_type, vector_in, matrix_in), + _vector_size(vector_size), + _result_size(result_size), + _common_inner(common_inner) +{ +} eval::InterpretedFunction::Instruction DenseXWProductFunction::compile_self(Stash &stash) const { - Self &self = stash.create<Self>(result_type(), _vectorSize, _resultSize); + Self &self = stash.create<Self>(result_type(), _vector_size, _result_size); auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _commonDimensionInnermost); + rhs().result_type().cell_type(), _common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } @@ -172,9 +171,9 @@ void DenseXWProductFunction::visit_self(vespalib::ObjectVisitor &visitor) const { Super::visit_self(visitor); - visitor.visitInt("vector_size", _vectorSize); - visitor.visitInt("result_size", _resultSize); - visitor.visitBool("common_dimension_innermost", _commonDimensionInnermost); + visitor.visitInt("vector_size", _vector_size); + visitor.visitInt("result_size", _result_size); + visitor.visitBool("common_inner", _common_inner); } const TensorFunction & diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h index f2f4d67c0f0..d7c39fa45a2 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h @@ -4,7 +4,6 @@ #include <vespa/eval/eval/tensor_function.h> #include "dense_tensor_view.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace vespalib::tensor { @@ -16,37 +15,34 @@ class DenseXWProductFunction : public eval::tensor_function::Op2 using Super = eval::tensor_function::Op2; public: struct Self { - const eval::ValueType _resultType; - const size_t _vectorSize; - const size_t _resultSize; - hwaccelrated::IAccelrated::UP _hwAccelerator; - Self(const eval::ValueType &resultType, - size_t vectorSize, - size_t resultSize); - ~Self() {} + eval::ValueType result_type; + size_t vector_size; + size_t result_size; + Self(const eval::ValueType &result_type_in, + size_t vector_size_in, size_t result_size_in); + ~Self(); }; private: - const size_t _vectorSize; - const size_t _resultSize; - bool _commonDimensionInnermost; + size_t _vector_size; + size_t _result_size; + bool _common_inner; public: - DenseXWProductFunction(const eval::ValueType &resultType, + DenseXWProductFunction(const eval::ValueType &result_type, const eval::TensorFunction &vector_in, const eval::TensorFunction &matrix_in, - size_t vectorSize, - size_t resultSize, - bool matrixHasCommonDimensionInnermost); + size_t vector_size, + size_t result_size, + bool common_inner); ~DenseXWProductFunction() {} bool result_is_mutable() const override { return true; } - size_t vectorSize() const { return _vectorSize; } - size_t resultSize() const { return _resultSize; } - - bool matrixHasCommonDimensionInnermost() const { return _commonDimensionInnermost; } + size_t vector_size() const { return _vector_size; } + size_t result_size() const { return _result_size; } + bool common_inner() const { return _common_inner; } eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index 002d2770dcd..99134a6e297 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "httpclient.h" +#include <vespa/vespalib/net/socket_spec.h> #include <cassert> #include <cstring> @@ -69,7 +70,8 @@ HTTPClient::connect_socket() if (!handle.valid()) { return false; } - _socket = vespalib::SyncCryptoSocket::create(*_engine, std::move(handle), false); + _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), + vespalib::SocketSpec::from_host_port(_hostname, _port)); return bool(_socket); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9535b62b15c..fc54c61fca8 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -55,6 +55,18 @@ public class Flags { "Whether to enable Nessus.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag ENABLE_FLEET_SSHD_CONFIG = defineFeatureFlag( + "enable-fleet-sshd-config", false, + "Whether fleet should manage the /etc/ssh/sshd_config file.", + "Takes effect on next host admin tick.", + HOSTNAME); + + public static final UnboundBooleanFlag FLEET_CANARY = defineFeatureFlag( + "fleet-canary", false, + "Whether the host is a fleet canary.", + "Takes effect on next host admin tick.", + HOSTNAME); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -90,17 +102,6 @@ public class Flags { "Takes effect on the next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag INCLUDE_SIS_IN_TRUSTSTORE = defineFeatureFlag( - "include-sis-in-truststore", false, - "Whether to use the trust store backed by Athenz and (in public) Service Identity certificates in " + - "host-admin and/or Docker containers", - "Takes effect on restart of host-admin (for host-admin), and restart of Docker container.", - // For host-admin, HOSTNAME and NODE_TYPE is available - // For Docker containers, HOSTNAME and APPLICATION_ID is available - // WARNING: Having different sets of dimensions is DISCOURAGED in general, but needed for here since - // trust store for host-admin is determined before having access to application ID from node repo. - HOSTNAME, NODE_TYPE, APPLICATION_ID); - public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag( "tls-insecure-mixed-mode", "tls_client_mixed_server", "TLS insecure mixed mode. Allowed values: ['plaintext_client_mixed_server', 'tls_client_mixed_server', 'tls_client_tls_server']", @@ -125,6 +126,21 @@ public class Flags { "scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).", "Takes effect on next run of NodeRebooter"); + public static final UnboundBooleanFlag ENABLE_LARGE_ORCHESTRATOR_LOCKS = defineFeatureFlag( + "enable-large-orchestrator-locks", true, + "If enabled, the orchestrator will accumulate application locks during probe in batch suspension, " + + "and release them in reverse order only after the non-probe is complete. Can be set depending on " + + "parent hostname.", + "Takes immediate effect for new batch suspensions.", + HOSTNAME); + + public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag( + "retire-with-permanently-down", false, + "If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " + + "instead of ALLOWED_TO_BE_DOWN (old behavior).", + "Takes effect on the next run of RetiredExpirer.", + HOSTNAME); + public static final UnboundBooleanFlag ENABLE_DYNAMIC_PROVISIONING = defineFeatureFlag( "enable-dynamic-provisioning", false, "Provision a new docker host when we otherwise can't allocate a docker node", @@ -173,13 +189,6 @@ public class Flags { "Takes effect on restart of process", NODE_TYPE, HOSTNAME); - public static final UnboundBooleanFlag USE_OLD_METRICS_CHECKS = defineFeatureFlag( - "use-old-metrics-checks", true, - "Whether to use old metrics checks", - "Takes effect on next host admin tick", - NODE_TYPE, HOSTNAME, APPLICATION_ID); - - public static final UnboundBooleanFlag ENABLE_DISK_WRITE_TEST = defineFeatureFlag( "enable-disk-write-test", false, "Regularly issue a small write to disk and fail the host if it is not successful", @@ -191,40 +200,45 @@ public class Flags { "Whether to disable CM3.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag USE_4443_UPSTREAM = defineFeatureFlag( - "use-4443-upstream", false, - "Use port 4443 for nginx upstream", - "Takes effect when routing container asks for new config", - APPLICATION_ID); + public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( + "generate-l4-routing-config", false, + "Whether routing nodes should generate L4 routing config", + "Takes effect immediately", + ZONE_ID, HOSTNAME); - public static final UnboundBooleanFlag ENABLE_IN_PLACE_RESIZE = defineFeatureFlag( - "enable-in-place-resize", false, - "Whether nodes can be resized in-place when certain conditions are met", - "Takes effect on next deployment", + public static final UnboundBooleanFlag USE_REFRESHED_ENDPOINT_CERTIFICATE = defineFeatureFlag( + "use-refreshed-endpoint-certificate", false, + "Whether an application should start using a newer certificate/key pair if available", + "Takes effect on the next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag FAIL_STARTING_NODE_ON_IP_MISMATCH = defineFeatureFlag( - "fail-starting-node-on-ip-mismatch", false, - "Whether node-admin should refuse to start container when there is an IP mismatch between the DNS and node-repository", - "Takes effect on next node creation (f.ex. node reboot or vespa version upgrade)"); + public static final UnboundStringFlag ENDPOINT_CERTIFICATE_BACKFILL = defineStringFlag( + "endpoint-certificate-backfill", "disable", + "Whether the endpoint certificate maintainer should backfill missing certificate data from cameo", + "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\"" + ); - public static final UnboundBooleanFlag USE_CONFIG_SERVER_FOR_TESTER_API_CALLS = defineFeatureFlag( - "use-config-server-for-tester-api-calls", false, - "Whether controller should send requests to tester API through config server (if false) or tester endpoint (if true)", - "Takes effect immediately", - ZONE_ID); + public static final UnboundBooleanFlag USE_NEW_ATHENZ_FILTER = defineFeatureFlag( + "use-new-athenz-filter", false, + "Use new Athenz filter that supports access-tokens", + "Takes effect at redeployment", + APPLICATION_ID); - public static final UnboundBooleanFlag INSTALL_L4_ROUTING_SUPPORT = defineFeatureFlag( - "install-l4-routing-support", false, - "Whether routing nodes should install package supporting L4 routing", - "Takes effect immediately", - ZONE_ID, HOSTNAME); + public static final UnboundBooleanFlag USE_RPM_PACKAGES_FOR_DATA_HIGHWAY = defineFeatureFlag( + "use-rpm-packages-for-data-highway", false, + "Whether RPM packages should be used for Data Highway", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( - "generate-l4-routing-config", false, - "Whether routing nodes should generate L4 routing config", - "Takes effect immediately", - ZONE_ID, HOSTNAME); + public static final UnboundStringFlag DOCKER_IMAGE_OVERRIDE = defineStringFlag( + "docker-image-override", "", + "Override the Docker image to use for deployments. This must containing the image name only, without tag", + "Takes effect on next host-admin tick", APPLICATION_ID); + + public static final UnboundBooleanFlag ENDPOINT_CERT_IN_SHARED_ROUTING = defineFeatureFlag( + "endpoint-cert-in-shared-routing", false, + "Whether to provision and use endpoint certs for apps in shared routing zones", + "Takes effect on next deployment of the application", APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index b70b3fa8b01..d94b6759077 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -65,7 +65,11 @@ struct BlockingCryptoEngine : public CryptoEngine { Gate handshake_work_enter; Gate handshake_work_exit; Gate handshake_socket_deleted; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool) override { + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override { + return std::make_unique<BlockingCryptoSocket>(std::move(socket), + handshake_work_enter, handshake_work_exit, handshake_socket_deleted); + } + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override { return std::make_unique<BlockingCryptoSocket>(std::move(socket), handshake_work_enter, handshake_work_exit, handshake_socket_deleted); } diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 74a7d19387c..c5afd627a5a 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -9,10 +9,13 @@ #include "config.h" #include "transport_thread.h" #include "transport.h" +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/log/log.h> LOG_SETUP(".fnet"); +std::atomic<uint64_t> FNET_Connection::_num_connections = 0; + namespace { class SyncPacket : public FNET_DummyPacket { private: @@ -470,7 +473,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _adminChannel(nullptr), - _socket(owner->owner().create_crypto_socket(std::move(socket), true)), + _socket(owner->owner().create_server_crypto_socket(std::move(socket))), _resolve_handler(nullptr), _context(), _state(FNET_CONNECTING), @@ -489,6 +492,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _cleanup(nullptr) { assert(_socket && (_socket->get_fd() >= 0)); + _num_connections.fetch_add(1, std::memory_order_relaxed); } @@ -526,6 +530,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _adminChannel = admin.get(); _channels.Register(admin.release()); } + _num_connections.fetch_add(1, std::memory_order_relaxed); } @@ -536,6 +541,7 @@ FNET_Connection::~FNET_Connection() delete _adminChannel; } assert(_cleanup == nullptr); + _num_connections.fetch_sub(1, std::memory_order_relaxed); } @@ -574,7 +580,7 @@ FNET_Connection::handle_add_event() { if (_resolve_handler) { auto tweak = [this](vespalib::SocketHandle &handle) { return Owner()->tune(handle); }; - _socket = Owner()->owner().create_crypto_socket(_resolve_handler->address.connect(tweak), false); + _socket = Owner()->owner().create_client_crypto_socket(_resolve_handler->address.connect(tweak), vespalib::SocketSpec(GetSpec())); _ioc_socket_fd = _socket->get_fd(); _resolve_handler.reset(); } diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 760b1b96d4d..c9c49c5151a 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -10,6 +10,7 @@ #include <vespa/vespalib/net/socket_handle.h> #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/crypto_socket.h> +#include <atomic> class FNET_IPacketStreamer; class FNET_IServerAdapter; @@ -111,6 +112,8 @@ private: FNET_IConnectionCleanupHandler *_cleanup; // cleanup handler + static std::atomic<uint64_t> _num_connections; // total number of connections + FNET_Connection(const FNET_Connection &); FNET_Connection &operator=(const FNET_Connection &); @@ -517,5 +520,10 @@ public: */ uint32_t getInputBufferSize() const { return _input.GetBufSize(); } + /** + * @return the total number of connection objects + **/ + static uint64_t get_num_connections() { + return _num_connections.load(std::memory_order_relaxed); + } }; - diff --git a/fnet/src/vespa/fnet/databuffer.cpp b/fnet/src/vespa/fnet/databuffer.cpp index 74a8bc4e12c..d982f9609e0 100644 --- a/fnet/src/vespa/fnet/databuffer.cpp +++ b/fnet/src/vespa/fnet/databuffer.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "databuffer.h" +#include <cstdio> FNET_DataBuffer::FNET_DataBuffer(uint32_t len) : _bufstart(nullptr), diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index f9cbfe2a662..a8cb7884534 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -409,7 +409,7 @@ FRT_Supervisor::SchedulerPtr::SchedulerPtr(FNET_TransportThread *transport_threa namespace fnet::frt { StandaloneFRT::StandaloneFRT() - : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), + : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)), _transport(std::make_unique<FNET_Transport>()), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { @@ -417,7 +417,7 @@ StandaloneFRT::StandaloneFRT() } StandaloneFRT::StandaloneFRT(vespalib::CryptoEngine::SP crypto) - : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)), + : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)), _transport(std::make_unique<FNET_Transport>(std::move(crypto), 1)), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index 28e645d9e03..d3b52969c8c 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -54,9 +54,15 @@ FNET_Transport::resolve_async(const vespalib::string &spec, } vespalib::CryptoSocket::UP -FNET_Transport::create_crypto_socket(vespalib::SocketHandle socket, bool is_server) +FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec) { - return _crypto_engine->create_crypto_socket(std::move(socket), is_server); + return _crypto_engine->create_client_crypto_socket(std::move(socket), spec); +} + +vespalib::CryptoSocket::UP +FNET_Transport::create_server_crypto_socket(vespalib::SocketHandle socket) +{ + return _crypto_engine->create_server_crypto_socket(std::move(socket)); } FNET_TransportThread * diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 8d1ba48c1b0..02ef22c7fb6 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -79,17 +79,25 @@ public: vespalib::AsyncResolver::ResultHandler::WP result_handler); /** - * Wrap a plain socket endpoint in a CryptoSocket. The + * Wrap a plain socket endpoint (client side) in a CryptoSocket. The * implementation will be determined by the CryptoEngine used by * this Transport. * * @return socket abstraction able to perform encryption and decryption * @param socket low-level socket - * @param is_server which end of the connection the socket - * represents. This is needed to support - * asymmetrical handshaking. + * @param spec who we are connecting to **/ - vespalib::CryptoSocket::UP create_crypto_socket(vespalib::SocketHandle socket, bool is_server); + vespalib::CryptoSocket::UP create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec); + + /** + * Wrap a plain socket endpoint (server side) in a CryptoSocket. The + * implementation will be determined by the CryptoEngine used by + * this Transport. + * + * @return socket abstraction able to perform encryption and decryption + * @param socket low-level socket + **/ + vespalib::CryptoSocket::UP create_server_crypto_socket(vespalib::SocketHandle socket); /** * Select one of the underlying transport threads. The selection diff --git a/functions.cmake b/functions.cmake index fc6fbc0b146..43dd2d30853 100644 --- a/functions.cmake +++ b/functions.cmake @@ -558,7 +558,7 @@ function(install_fat_java_artifact NAME) endfunction() function(install_absolute_symlink TARGET LINK) - install(CODE "execute_process(COMMAND ln -snf ${TARGET} \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${LINK})") + install(CODE "execute_process(COMMAND ln -snf ${TARGET} \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/${LINK})") endfunction(install_absolute_symlink) function(install_symlink TARGET LINK) diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java new file mode 100644 index 00000000000..a97024df95c --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import java.time.Duration; + +/** + * An abstraction that calculates the next delay based on the current retry count. + * + * @author bjorncs + */ +@FunctionalInterface +interface DelaySupplier { + Duration getDelay(int executionCount); + + class Fixed implements DelaySupplier { + private final Duration delay; + + Fixed(Duration delay) { + this.delay = delay; + } + + @Override + public Duration getDelay(int executionCount) { return delay; } + } + + class Exponential implements DelaySupplier { + private final Duration startDelay; + private final Duration maxDelay; + + Exponential(Duration startDelay, Duration maxDelay) { + this.startDelay = startDelay; + this.maxDelay = maxDelay; + } + + @Override + public Duration getDelay(int executionCount) { + Duration nextDelay = startDelay; + for (int i = 1; i < executionCount; ++i) { + nextDelay = nextDelay.multipliedBy(2); + } + return maxDelay.compareTo(nextDelay) > 0 ? nextDelay : maxDelay; + } + } +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java new file mode 100644 index 00000000000..dca1907c08b --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java @@ -0,0 +1,126 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import org.apache.http.annotation.Contract; +import org.apache.http.annotation.ThreadingBehavior; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.protocol.HttpContext; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.function.Predicate; +import java.util.logging.Logger; + +/** + * A {@link HttpRequestRetryHandler} that supports delayed retries. + * + * @author bjorncs + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public class DelayedConnectionLevelRetryHandler implements HttpRequestRetryHandler { + + private static final Logger log = Logger.getLogger(HttpRequestRetryHandler.class.getName()); + + private final DelaySupplier delaySupplier; + private final int maxRetries; + private final RetryPredicate<IOException> predicate; + private final RetryConsumer<IOException> retryConsumer; + private final RetryFailedConsumer<IOException> retryFailedConsumer; + private final Sleeper sleeper; + + private DelayedConnectionLevelRetryHandler( + DelaySupplier delaySupplier, + int maxRetries, + RetryPredicate<IOException> predicate, + RetryConsumer<IOException> retryConsumer, + RetryFailedConsumer<IOException> retryFailedConsumer, + Sleeper sleeper) { + this.delaySupplier = delaySupplier; + this.maxRetries = maxRetries; + this.predicate = predicate; + this.retryConsumer = retryConsumer; + this.retryFailedConsumer = retryFailedConsumer; + this.sleeper = sleeper; + } + + @Override + public boolean retryRequest(IOException exception, int executionCount, HttpContext ctx) { + log.fine(() -> String.format("retryRequest(exception='%s', executionCount='%d', ctx='%s'", + exception.getClass().getName(), executionCount, ctx)); + HttpClientContext clientCtx = HttpClientContext.adapt(ctx); + if (!predicate.test(exception, clientCtx)) { + log.fine(() -> String.format("Not retrying for '%s'", ctx)); + return false; + } + if (executionCount > maxRetries) { + log.fine(() -> String.format("Max retries exceeded for '%s'", ctx)); + retryFailedConsumer.onRetryFailed(exception, executionCount, clientCtx); + return false; + } + Duration delay = delaySupplier.getDelay(executionCount); + log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx)); + retryConsumer.onRetry(exception, delay, executionCount, clientCtx); + sleeper.sleep(delay); + return true; + } + + public static class Builder { + + private final DelaySupplier delaySupplier; + private final int maxRetries; + private RetryPredicate<IOException> predicate = (ioException, ctx) -> true; + private RetryConsumer<IOException> retryConsumer = (exception, delay, count, ctx) -> {}; + private RetryFailedConsumer<IOException> retryFailedConsumer = (exception, count, ctx) -> {}; + private Sleeper sleeper = new Sleeper.Default(); + + private Builder(DelaySupplier delaySupplier, int maxRetries) { + this.delaySupplier = delaySupplier; + this.maxRetries = maxRetries; + } + + public static Builder withFixedDelay(Duration delay, int maxRetries) { + return new Builder(new DelaySupplier.Fixed(delay), maxRetries); + } + + public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) { + return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries); + } + + public Builder retryForExceptions(List<Class<? extends IOException>> exceptionTypes) { + this.predicate = (ioException, ctx) -> exceptionTypes.stream().anyMatch(type -> type.isInstance(ioException)); + return this; + } + + public Builder retryForExceptions(Predicate<IOException> predicate) { + this.predicate = (ioException, ctx) -> predicate.test(ioException); + return this; + } + + public Builder retryFor(RetryPredicate<IOException> predicate) { + this.predicate = predicate; + return this; + } + + public Builder onRetry(RetryConsumer<IOException> consumer) { + this.retryConsumer = consumer; + return this; + } + + public Builder onRetryFailed(RetryFailedConsumer<IOException> consumer) { + this.retryFailedConsumer = consumer; + return this; + } + + // For unit testing + Builder withSleeper(Sleeper sleeper) { + this.sleeper = sleeper; + return this; + } + + public DelayedConnectionLevelRetryHandler build() { + return new DelayedConnectionLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer, sleeper); + } + } +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java new file mode 100644 index 00000000000..041b501809c --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java @@ -0,0 +1,125 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import org.apache.http.HttpResponse; +import org.apache.http.annotation.Contract; +import org.apache.http.annotation.ThreadingBehavior; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.protocol.HttpContext; + +import java.time.Duration; +import java.util.List; +import java.util.function.Predicate; +import java.util.logging.Logger; + +/** + * A {@link ServiceUnavailableRetryStrategy} that supports delayed retries on any response types. + * + * @author bjorncs + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public class DelayedResponseLevelRetryHandler implements ServiceUnavailableRetryStrategy { + + private static final Logger log = Logger.getLogger(DelayedResponseLevelRetryHandler.class.getName()); + + private final DelaySupplier delaySupplier; + private final int maxRetries; + private final RetryPredicate<HttpResponse> predicate; + private final RetryConsumer<HttpResponse> retryConsumer; + private final RetryFailedConsumer<HttpResponse> retryFailedConsumer; + private final ThreadLocal<Long> retryInterval = ThreadLocal.withInitial(() -> 0L); + + private DelayedResponseLevelRetryHandler( + DelaySupplier delaySupplier, + int maxRetries, + RetryPredicate<HttpResponse> predicate, + RetryConsumer<HttpResponse> retryConsumer, + RetryFailedConsumer<HttpResponse> retryFailedConsumer) { + + this.delaySupplier = delaySupplier; + this.maxRetries = maxRetries; + this.predicate = predicate; + this.retryConsumer = retryConsumer; + this.retryFailedConsumer = retryFailedConsumer; + } + + @Override + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext ctx) { + log.fine(() -> String.format("retryRequest(responseCode='%s', executionCount='%d', ctx='%s'", + response.getStatusLine().getStatusCode(), executionCount, ctx)); + HttpClientContext clientCtx = HttpClientContext.adapt(ctx); + if (!predicate.test(response, clientCtx)) { + log.fine(() -> String.format("Not retrying for '%s'", ctx)); + return false; + } + if (executionCount > maxRetries) { + log.fine(() -> String.format("Max retries exceeded for '%s'", ctx)); + retryFailedConsumer.onRetryFailed(response, executionCount, clientCtx); + return false; + } + Duration delay = delaySupplier.getDelay(executionCount); + log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx)); + retryInterval.set(delay.toMillis()); + retryConsumer.onRetry(response, delay, executionCount, clientCtx); + return true; + } + + @Override + public long getRetryInterval() { + // Calls to getRetryInterval are always guarded by a call to retryRequest (using the same thread). + // A thread local allows this retry handler to be thread safe and support dynamic retry intervals + return retryInterval.get(); + } + + public static class Builder { + + private final DelaySupplier delaySupplier; + private final int maxRetries; + private RetryPredicate<HttpResponse> predicate = (response, ctx) -> true; + private RetryConsumer<HttpResponse> retryConsumer = (response, delay, count, ctx) -> {}; + private RetryFailedConsumer<HttpResponse> retryFailedConsumer = (response, count, ctx) -> {}; + + private Builder(DelaySupplier delaySupplier, int maxRetries) { + this.delaySupplier = delaySupplier; + this.maxRetries = maxRetries; + } + + public static Builder withFixedDelay(Duration delay, int maxRetries) { + return new Builder(new DelaySupplier.Fixed(delay), maxRetries); + } + + public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) { + return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries); + } + + public Builder retryForStatusCodes(List<Integer> statusCodes) { + this.predicate = (response, ctx) -> statusCodes.contains(response.getStatusLine().getStatusCode()); + return this; + } + + public Builder retryForResponses(Predicate<HttpResponse> predicate) { + this.predicate = (response, ctx) -> predicate.test(response); + return this; + } + + public Builder retryFor(RetryPredicate<HttpResponse> predicate) { + this.predicate = predicate; + return this; + } + + public Builder onRetry(RetryConsumer<HttpResponse> consumer) { + this.retryConsumer = consumer; + return this; + } + + public Builder onRetryFailed(RetryFailedConsumer<HttpResponse> consumer) { + this.retryFailedConsumer = consumer; + return this; + } + + public DelayedResponseLevelRetryHandler build() { + return new DelayedResponseLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer); + } + } +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java new file mode 100644 index 00000000000..494be051673 --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java @@ -0,0 +1,16 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import org.apache.http.client.protocol.HttpClientContext; + +import java.time.Duration; + +/** + * Invoked before performing a delay and retry. + * + * @author bjorncs + */ +@FunctionalInterface +public interface RetryConsumer<T> { + void onRetry(T data, Duration delay, int executionCount, HttpClientContext context); +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java new file mode 100644 index 00000000000..ed326ac1210 --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java @@ -0,0 +1,14 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import org.apache.http.client.protocol.HttpClientContext; + +/** + * Invoked after the last retry has failed. + * + * @author bjorncs + */ +@FunctionalInterface +public interface RetryFailedConsumer<T> { + void onRetryFailed(T response, int executionCount, HttpClientContext context); +} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java new file mode 100644 index 00000000000..ccf62c9be9f --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java @@ -0,0 +1,13 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import org.apache.http.client.protocol.HttpClientContext; + +import java.util.function.BiPredicate; + +/** + * A predicate that determines whether an operation should be retried. + * + * @author bjorncs + */ +public interface RetryPredicate<T> extends BiPredicate<T, HttpClientContext> {} diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java new file mode 100644 index 00000000000..06a7359f307 --- /dev/null +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java @@ -0,0 +1,26 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import java.time.Duration; + +/** + * An abstraction used for mocking {@link Thread#sleep(long)} in unit tests. + * + * @author bjorncs + */ +interface Sleeper { + void sleep(Duration duration); + + class Default implements Sleeper { + @Override + public void sleep(Duration duration) { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } +} + diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java new file mode 100644 index 00000000000..85adeae6d78 --- /dev/null +++ b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java @@ -0,0 +1,134 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; + +import com.yahoo.vespa.jdk8compat.List; +import org.apache.http.client.protocol.HttpClientContext; +import org.junit.Test; + +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.net.ConnectException; +import java.time.Duration; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +public class DelayedConnectionLevelRetryHandlerTest { + + @SuppressWarnings("unchecked") + @Test + public void retry_consumers_are_invoked() { + RetryConsumer<IOException> retryConsumer = (RetryConsumer<IOException>) mock(RetryConsumer.class); + RetryFailedConsumer<IOException> retryFailedConsumer = (RetryFailedConsumer<IOException>) mock(RetryFailedConsumer.class); + + Duration delay = Duration.ofSeconds(10); + int maxRetries = 5; + + DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder + .withFixedDelay(delay, maxRetries) + .withSleeper(mock(Sleeper.class)) + .onRetry(retryConsumer) + .onRetryFailed(retryFailedConsumer) + .build(); + + IOException exception = new IOException(); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(exception, i, ctx); + } + + verify(retryFailedConsumer).onRetryFailed(exception, lastExecutionCount, ctx); + for (int i = 1; i < lastExecutionCount; i++) { + verify(retryConsumer).onRetry(exception, delay, i, ctx); + } + } + + @Test + public void retry_with_fixed_delay_sleeps_for_expected_duration() { + Sleeper sleeper = mock(Sleeper.class); + + Duration delay = Duration.ofSeconds(2); + int maxRetries = 2; + + DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder + .withFixedDelay(delay, maxRetries) + .withSleeper(sleeper) + .build(); + + IOException exception = new IOException(); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(exception, i, ctx); + } + + verify(sleeper, times(2)).sleep(delay); + } + + @Test + public void retry_with_fixed_backoff_sleeps_for_expected_durations() { + Sleeper sleeper = mock(Sleeper.class); + + Duration startDelay = Duration.ofMillis(500); + Duration maxDelay = Duration.ofSeconds(5); + int maxRetries = 10; + + DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder + .withExponentialBackoff(startDelay, maxDelay, maxRetries) + .withSleeper(sleeper) + .build(); + + IOException exception = new IOException(); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(exception, i, ctx); + } + + verify(sleeper).sleep(startDelay); + verify(sleeper).sleep(Duration.ofSeconds(1)); + verify(sleeper).sleep(Duration.ofSeconds(2)); + verify(sleeper).sleep(Duration.ofSeconds(4)); + verify(sleeper, times(6)).sleep(Duration.ofSeconds(5)); + } + + @Test + public void retries_for_listed_exceptions_until_max_retries_exceeded() { + int maxRetries = 2; + + DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder + .withFixedDelay(Duration.ofSeconds(2), maxRetries) + .retryForExceptions(List.of(SSLException.class, ConnectException.class)) + .withSleeper(mock(Sleeper.class)) + .build(); + + SSLException sslException = new SSLException("ssl error"); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i < lastExecutionCount; i++) { + assertTrue(handler.retryRequest(sslException, i, ctx)); + } + assertFalse(handler.retryRequest(sslException, lastExecutionCount, ctx)); + } + + @Test + public void does_not_retry_for_non_listed_exception() { + DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder + .withFixedDelay(Duration.ofSeconds(2), 2) + .retryForExceptions(List.of(SSLException.class, ConnectException.class)) + .withSleeper(mock(Sleeper.class)) + .build(); + + IOException ioException = new IOException(); + HttpClientContext ctx = new HttpClientContext(); + assertFalse(handler.retryRequest(ioException, 1, ctx)); + } + +}
\ No newline at end of file diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java new file mode 100644 index 00000000000..7be1e143078 --- /dev/null +++ b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java @@ -0,0 +1,129 @@ +package ai.vespa.util.http.retry;// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +public class DelayedResponseLevelRetryHandlerTest { + + @Test + @SuppressWarnings("unchecked") + public void retry_consumers_are_invoked() { + RetryConsumer<HttpResponse> retryConsumer = mock(RetryConsumer.class); + RetryFailedConsumer<HttpResponse> retryFailedConsumer = mock(RetryFailedConsumer.class); + + Duration delay = Duration.ofSeconds(10); + int maxRetries = 5; + + DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder + .withFixedDelay(delay, maxRetries) + .onRetry(retryConsumer) + .onRetryFailed(retryFailedConsumer) + .build(); + + HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(response, i, ctx); + } + + verify(retryFailedConsumer).onRetryFailed(response, lastExecutionCount, ctx); + for (int i = 1; i < lastExecutionCount; i++) { + verify(retryConsumer).onRetry(response, delay, i, ctx); + } + } + + @Test + public void retry_with_fixed_delay_sleeps_for_expected_duration() { + Duration delay = Duration.ofSeconds(2); + int maxRetries = 2; + + DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder + .withFixedDelay(delay, maxRetries) + .build(); + + HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(response, i, ctx); + assertThat(handler.getRetryInterval()).isEqualTo(delay.toMillis()); + } + } + + @Test + public void retry_with_fixed_backoff_sleeps_for_expected_durations() { + Duration startDelay = Duration.ofMillis(500); + Duration maxDelay = Duration.ofSeconds(5); + int maxRetries = 10; + + DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder + .withExponentialBackoff(startDelay, maxDelay, maxRetries) + .build(); + + HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + List<Duration> expectedIntervals = + com.yahoo.vespa.jdk8compat.List.of( + startDelay, Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4), + Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), + Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5)); + for (int i = 1; i <= lastExecutionCount; i++) { + handler.retryRequest(response, i, ctx); + assertThat(handler.getRetryInterval()).isEqualTo(expectedIntervals.get(i-1).toMillis()); + } + } + + @Test + public void retries_for_listed_exceptions_until_max_retries_exceeded() { + int maxRetries = 2; + + DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder + .withFixedDelay(Duration.ofSeconds(2), maxRetries) + .retryForStatusCodes(com.yahoo.vespa.jdk8compat.List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) + .build(); + + HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE); + HttpClientContext ctx = new HttpClientContext(); + int lastExecutionCount = maxRetries + 1; + for (int i = 1; i < lastExecutionCount; i++) { + assertTrue(handler.retryRequest(response, i, ctx)); + } + assertFalse(handler.retryRequest(response, lastExecutionCount, ctx)); + } + + @Test + public void does_not_retry_for_non_listed_exception() { + DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder + .withFixedDelay(Duration.ofSeconds(2), 2) + .retryForStatusCodes(com.yahoo.vespa.jdk8compat.List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) + .build(); + + HttpResponse response = createResponse(HttpStatus.SC_OK); + HttpClientContext ctx = new HttpClientContext(); + assertFalse(handler.retryRequest(response, 1, ctx)); + } + + private static HttpResponse createResponse(int statusCode) { + return new BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, statusCode, "reason phrase")); + } + +}
\ No newline at end of file diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 74e0ee36959..f3d9fa9583c 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -2,26 +2,32 @@ package com.yahoo.jdisc.http.filter.security.athenz; import com.google.inject.Inject; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.CredentialsToVerify; import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzResourceName; -import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.tls.AthenzX509CertificateUtils; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.athenz.zpe.AuthorizationResult; import com.yahoo.vespa.athenz.zpe.DefaultZpe; import com.yahoo.vespa.athenz.zpe.Zpe; import java.security.cert.X509Certificate; +import java.util.EnumSet; +import java.util.List; import java.util.Optional; -import java.util.function.Function; +import java.util.logging.Logger; -import static java.util.Collections.singletonList; +import static com.yahoo.jdisc.Response.Status.FORBIDDEN; +import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ACCESS_TOKEN; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ROLE_CERTIFICATE; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ROLE_TOKEN; /** * An Athenz security filter that uses a configured action and resource name to control access. @@ -30,116 +36,148 @@ import static java.util.Collections.singletonList; */ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { - private final String headerName; + private static final String ATTRIBUTE_PREFIX = "jdisc-security-filters.athenz-authorization-filter"; + public static final String RESULT_ATTRIBUTE = ATTRIBUTE_PREFIX + ".result"; + public static final String MATCHED_ROLE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".matched-role"; + public static final String IDENTITY_NAME_ATTRIBUTE = ATTRIBUTE_PREFIX + ".identity-name"; + public static final String MATCHED_CREDENTIAL_TYPE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".credentials-type"; + + private static final Logger log = Logger.getLogger(AthenzAuthorizationFilter.class.getName()); + + private final String roleTokenHeaderName; + private final EnumSet<EnabledCredentials.Enum> enabledCredentials; private final Zpe zpe; private final RequestResourceMapper requestResourceMapper; - private final CredentialsToVerify.Enum credentialsToVerify; @Inject public AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config, RequestResourceMapper resourceMapper) { this(config, resourceMapper, new DefaultZpe()); } - AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config, - RequestResourceMapper resourceMapper, - Zpe zpe) { - this.headerName = config.roleTokenHeaderName(); - this.credentialsToVerify = config.credentialsToVerify(); + public AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config, + RequestResourceMapper resourceMapper, + Zpe zpe) { + this.roleTokenHeaderName = config.roleTokenHeaderName(); + List<EnabledCredentials.Enum> enabledCredentials = config.enabledCredentials(); + this.enabledCredentials = enabledCredentials.isEmpty() + ? EnumSet.allOf(EnabledCredentials.Enum.class) + : EnumSet.copyOf(enabledCredentials); this.requestResourceMapper = resourceMapper; this.zpe = zpe; } @Override - protected Optional<ErrorResponse> filter(DiscFilterRequest request) { - Optional<ResourceNameAndAction> resourceMapping = - requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString()); - if (!resourceMapping.isPresent()) { - return Optional.empty(); - } - Optional<X509Certificate> roleCertificate = getRoleCertificate(request); - Optional<ZToken> roleToken = getRoleToken(request, headerName); - switch (credentialsToVerify) { - case CERTIFICATE_ONLY: { - if (!roleCertificate.isPresent()) { - return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Missing client certificate")); - } - return checkAccessAllowed(roleCertificate.get(), resourceMapping.get(), request); - } - case TOKEN_ONLY: { - if (!roleToken.isPresent()) { - return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, - String.format("Role token header '%s' is missing or does not have a value.", headerName))); - } - return checkAccessAllowed(roleToken.get(), resourceMapping.get(), request); - } - case ANY: { - if (!roleCertificate.isPresent() && !roleToken.isPresent()) { - return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Both role token and role certificate is missing")); - } - if (roleCertificate.isPresent()) { - return checkAccessAllowed(roleCertificate.get(), resourceMapping.get(), request); - } else { - return checkAccessAllowed(roleToken.get(), resourceMapping.get(), request); - } + public Optional<ErrorResponse> filter(DiscFilterRequest request) { + try { + Optional<ResourceNameAndAction> resourceMapping = + requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString()); + log.log(LogLevel.DEBUG, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping)); + if (resourceMapping.isEmpty()) { + return Optional.empty(); } - default: { - throw new IllegalStateException("Unexpected mode: " + credentialsToVerify); + Result result = checkAccessAllowed(request, resourceMapping.get()); + AuthorizationResult.Type resultType = result.zpeResult.type(); + setAttribute(request, RESULT_ATTRIBUTE, resultType.name()); + if (resultType == AuthorizationResult.Type.ALLOW) { + populateRequestWithResult(request, result); + return Optional.empty(); } + log.log(LogLevel.DEBUG, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name())); + return Optional.of(new ErrorResponse(FORBIDDEN, "Access forbidden: " + resultType.getDescription())); + } catch (IllegalArgumentException e) { + log.log(LogLevel.DEBUG, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage())); + return Optional.of(new ErrorResponse(UNAUTHORIZED, e.getMessage())); } } - private static Optional<X509Certificate> getRoleCertificate(DiscFilterRequest request) { - return Optional.of(request.getClientCertificateChain()) - .filter(chain -> !chain.isEmpty()) - .map(chain -> chain.get(0)) - .filter(AthenzX509CertificateUtils::isAthenzRoleCertificate); + private Result checkAccessAllowed(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) { + // Note: the ordering of the if-constructs determines the precedence of the credential types + if (enabledCredentials.contains(ACCESS_TOKEN) + && isAccessTokenPresent(request) + && isClientCertificatePresent(request)) { + return checkAccessWithAccessToken(request, resourceAndAction); + } else if (enabledCredentials.contains(ROLE_CERTIFICATE) + && isClientCertificatePresent(request)) { + return checkAccessWithRoleCertificate(request, resourceAndAction); + } else if (enabledCredentials.contains(ROLE_TOKEN) + && isRoleTokenPresent(request)) { + return checkAccessWithRoleToken(request, resourceAndAction); + } else { + throw new IllegalArgumentException( + "Not authorized - request did not contain any of the allowed credentials: " + enabledCredentials); + } } - private static Optional<ZToken> getRoleToken(DiscFilterRequest request, String headerName) { - return Optional.ofNullable(request.getHeader(headerName)) - .filter(token -> !token.isEmpty()) - .map(ZToken::new); + private Result checkAccessWithAccessToken(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) { + AthenzAccessToken accessToken = getAccessToken(request); + X509Certificate identityCertificate = getClientCertificate(request); + var zpeResult = zpe.checkAccessAllowed( + accessToken, identityCertificate, resourceAndAction.resourceName(), resourceAndAction.action()); + return new Result(ACCESS_TOKEN, AthenzIdentities.from(identityCertificate), zpeResult); } - private Optional<ErrorResponse> checkAccessAllowed(X509Certificate certificate, - ResourceNameAndAction resourceNameAndAction, - DiscFilterRequest request) { - return checkAccessAllowed( - certificate, resourceNameAndAction, request, zpe::checkAccessAllowed, AthenzAuthorizationFilter::createPrincipal); + private Result checkAccessWithRoleCertificate(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) { + X509Certificate roleCertificate = getClientCertificate(request); + var zpeResult = zpe.checkAccessAllowed(roleCertificate, resourceAndAction.resourceName(), resourceAndAction.action()); + AthenzIdentity identity = AthenzX509CertificateUtils.getIdentityFromRoleCertificate(roleCertificate); + return new Result(ROLE_CERTIFICATE, identity, zpeResult); } - private Optional<ErrorResponse> checkAccessAllowed(ZToken roleToken, - ResourceNameAndAction resourceNameAndAction, - DiscFilterRequest request) { - return checkAccessAllowed( - roleToken, resourceNameAndAction, request, zpe::checkAccessAllowed, AthenzAuthorizationFilter::createPrincipal); + private Result checkAccessWithRoleToken(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) { + ZToken roleToken = getRoleToken(request); + var zpeResult = zpe.checkAccessAllowed(roleToken, resourceAndAction.resourceName(), resourceAndAction.action()); + return new Result(ROLE_TOKEN, roleToken.getIdentity(), zpeResult); } - private static <C> Optional<ErrorResponse> checkAccessAllowed(C credentials, - ResourceNameAndAction resAndAction, - DiscFilterRequest request, - ZpeCheck<C> accessCheck, - Function<C, AthenzPrincipal> principalFactory) { - AuthorizationResult authorizationResult = accessCheck.checkAccess(credentials, resAndAction.resourceName(), resAndAction.action()); - if (authorizationResult == AuthorizationResult.ALLOW) { - request.setUserPrincipal(principalFactory.apply(credentials)); - return Optional.empty(); - } - return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access forbidden: " + authorizationResult.getDescription())); + private static boolean isAccessTokenPresent(DiscFilterRequest request) { + return request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME) != null; + } + + private static boolean isClientCertificatePresent(DiscFilterRequest request) { + return !request.getClientCertificateChain().isEmpty(); } - private static AthenzPrincipal createPrincipal(X509Certificate certificate) { - AthenzIdentity identity = AthenzX509CertificateUtils.getIdentityFromRoleCertificate(certificate); - AthenzRole role = AthenzX509CertificateUtils.getRolesFromRoleCertificate(certificate); - return new AthenzPrincipal(identity, singletonList(role)); + private boolean isRoleTokenPresent(DiscFilterRequest request) { + return request.getHeader(roleTokenHeaderName) != null; } - private static AthenzPrincipal createPrincipal(ZToken roleToken) { - return new AthenzPrincipal(roleToken.getIdentity(), roleToken.getRoles()); + private static AthenzAccessToken getAccessToken(DiscFilterRequest request) { + return new AthenzAccessToken(request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME)); } - @FunctionalInterface private interface ZpeCheck<C> { - AuthorizationResult checkAccess(C credentials, AthenzResourceName resourceName, String action); + private static X509Certificate getClientCertificate(DiscFilterRequest request) { + return request.getClientCertificateChain().get(0); } + private ZToken getRoleToken(DiscFilterRequest request) { + return new ZToken(request.getHeader(roleTokenHeaderName)); + } + + private static void populateRequestWithResult(DiscFilterRequest request, Result result) { + request.setUserPrincipal( + new AthenzPrincipal(result.identity, result.zpeResult.matchedRole().map(List::of).orElse(List.of()))); + result.zpeResult.matchedRole().ifPresent(role -> { + request.setUserRoles(new String[]{role.roleName()}); + setAttribute(request, MATCHED_ROLE_ATTRIBUTE, role.roleName()); + }); + setAttribute(request, IDENTITY_NAME_ATTRIBUTE, result.identity.getFullName()); + setAttribute(request, MATCHED_CREDENTIAL_TYPE_ATTRIBUTE, result.credentialType.name()); + } + + private static void setAttribute(DiscFilterRequest request, String name, String value) { + log.log(LogLevel.DEBUG, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value)); + request.setAttribute(name, value); + } + + private static class Result { + final EnabledCredentials.Enum credentialType; + final AthenzIdentity identity; + final AuthorizationResult zpeResult; + + Result(EnabledCredentials.Enum credentialType, AthenzIdentity identity, AuthorizationResult zpeResult) { + this.credentialType = credentialType; + this.identity = identity; + this.zpeResult = zpeResult; + } + } } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java index 77709975cba..0bf000efc00 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java @@ -33,5 +33,13 @@ public interface RequestResourceMapper { public String action() { return action; } + + @Override + public String toString() { + return "ResourceNameAndAction{" + + "resourceName=" + resourceName + + ", action='" + action + '\'' + + '}'; + } } } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java index ec8a93019b0..e654182d665 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java @@ -10,9 +10,11 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.log.LogLevel; import java.io.UncheckedIOException; import java.util.Optional; +import java.util.logging.Logger; /** * A base class for {@link SecurityRequestFilter} implementations that renders an error response as JSON. @@ -21,22 +23,25 @@ import java.util.Optional; */ public abstract class JsonSecurityRequestFilterBase implements SecurityRequestFilter { + private static final Logger log = Logger.getLogger(JsonSecurityRequestFilterBase.class.getName()); private static final ObjectMapper mapper = new ObjectMapper(); @Override public final void filter(DiscFilterRequest request, ResponseHandler handler) { filter(request) - .ifPresent(errorResponse -> writeResponse(errorResponse, handler)); + .ifPresent(errorResponse -> writeResponse(request, errorResponse, handler)); } protected abstract Optional<ErrorResponse> filter(DiscFilterRequest request); - private void writeResponse(ErrorResponse error, ResponseHandler responseHandler) { + private void writeResponse(DiscFilterRequest request, ErrorResponse error, ResponseHandler responseHandler) { ObjectNode errorMessage = mapper.createObjectNode(); errorMessage.put("code", error.errorCode); errorMessage.put("message", error.message); error.response.headers().put("Content-Type", "application/json"); // Note: Overwrites header if already exists error.response.headers().put("Cache-Control", "must-revalidate,no-cache,no-store"); + log.log(LogLevel.DEBUG, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'", + request, error.response.getStatus(), error.errorCode, error.message)); try (FastContentWriter writer = ResponseDispatch.newInstance(error.response).connectFastWriter(responseHandler)) { writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage)); } catch (JsonProcessingException e) { diff --git a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def index c60b7a125f8..ab8c4a204df 100644 --- a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def +++ b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def @@ -2,7 +2,7 @@ namespace=jdisc.http.filter.security.athenz # Which credentials to verify. Note: ANY will prioritize token over certificate if both are present. -credentialsToVerify enum { CERTIFICATE_ONLY, TOKEN_ONLY, ANY } default=ANY +enabledCredentials[] enum { ROLE_CERTIFICATE, ROLE_TOKEN, ACCESS_TOKEN } -# Name of header which includes role token. Must be set if 'credentialsTypeRequired' is set to TOKEN_ONLY or ANY. -roleTokenHeaderName string default="" +# Name of role token http header +roleTokenHeaderName string default="Athenz-Role-Token" diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java index b81b26d458b..1fe8d73eb44 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java @@ -1,25 +1,44 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.athenz; -import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SubjectAlternativeName; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.athenz.zpe.AuthorizationResult; import com.yahoo.vespa.athenz.zpe.Zpe; import org.junit.Test; import org.mockito.Mockito; +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; import java.security.cert.X509Certificate; - -import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.CredentialsToVerify.Enum.ANY; -import static java.util.Collections.emptyList; -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.MATCHED_CREDENTIAL_TYPE_ATTRIBUTE; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.MATCHED_ROLE_ATTRIBUTE; +import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.RESULT_ATTRIBUTE; +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static com.yahoo.security.SubjectAlternativeName.Type.RFC822_NAME; +import static com.yahoo.vespa.athenz.zpe.AuthorizationResult.Type; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -28,76 +47,175 @@ import static org.mockito.Mockito.when; public class AthenzAuthorizationFilterTest { private static final AthenzResourceName RESOURCE_NAME = new AthenzResourceName("domain", "my-resource-name"); + private static final ZToken ROLE_TOKEN = new ZToken("v=Z1;d=domain;r=my-role;p=my-domain.my-service"); + private static final AthenzAccessToken ACCESS_TOKEN = new AthenzAccessToken("access-token"); + private static final AthenzIdentity IDENTITY = AthenzIdentities.from("user.john"); + private static final AthenzRole ROLE = new AthenzRole("my.domain", "my-role"); + private static final X509Certificate IDENTITY_CERTIFICATE = createDummyIdentityCertificate(IDENTITY); + private static final X509Certificate ROLE_CERTIFICATE = createDummyRoleCertificate(ROLE, IDENTITY); private static final String ACTION = "update"; private static final String HEADER_NAME = "Athenz-Role-Token"; - private static final AthenzAuthorizationFilterConfig CONFIG = createConfig(); - private static AthenzAuthorizationFilterConfig createConfig() { - return new AthenzAuthorizationFilterConfig( - new AthenzAuthorizationFilterConfig.Builder() - .roleTokenHeaderName(HEADER_NAME) - .credentialsToVerify(ANY)); + @Test + public void accepts_request_with_access_token() { + AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of()); + + MockResponseHandler responseHandler = new MockResponseHandler(); + DiscFilterRequest request = createRequest(null, ACCESS_TOKEN, IDENTITY_CERTIFICATE); + filter.filter(request, responseHandler); + + assertAuthorizationResult(request, Type.ALLOW); + assertRequestNotFiltered(responseHandler); + assertMatchedCredentialType(request, EnabledCredentials.ACCESS_TOKEN); + assertMatchedRole(request, ROLE); } @Test - public void accepts_valid_requests() { - AthenzAuthorizationFilter filter = - new AthenzAuthorizationFilter( - CONFIG, new StaticRequestResourceMapper(RESOURCE_NAME, ACTION), new AllowingZpe()); + public void accepts_request_with_role_certificate() { + AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of()); + + MockResponseHandler responseHandler = new MockResponseHandler(); + DiscFilterRequest request = createRequest(null, null, ROLE_CERTIFICATE); + filter.filter(request, responseHandler); + + assertAuthorizationResult(request, Type.ALLOW); + assertRequestNotFiltered(responseHandler); + assertMatchedCredentialType(request, EnabledCredentials.ROLE_CERTIFICATE); + assertMatchedRole(request, ROLE); + } + + @Test + public void accepts_request_with_role_token() { + AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of()); - RequestHandlerTestDriver.MockResponseHandler responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); - filter.filter(createRequest(), responseHandler); + MockResponseHandler responseHandler = new MockResponseHandler(); + DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null); + filter.filter(request, responseHandler); - assertNull(responseHandler.getResponse()); + assertAuthorizationResult(request, Type.ALLOW); + assertRequestNotFiltered(responseHandler); + assertMatchedCredentialType(request, EnabledCredentials.ROLE_TOKEN); + assertMatchedRole(request, ROLE); } @Test - public void returns_error_on_forbidden_requests() { + public void returns_unauthorized_for_request_with_disabled_credential_type() { AthenzAuthorizationFilter filter = - new AthenzAuthorizationFilter( - CONFIG, new StaticRequestResourceMapper(RESOURCE_NAME, ACTION), new DenyingZpe()); + createFilter(new AllowingZpe(), List.of(EnabledCredentials.ROLE_CERTIFICATE, EnabledCredentials.ACCESS_TOKEN)); - RequestHandlerTestDriver.MockResponseHandler responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); - filter.filter(createRequest(), responseHandler); + MockResponseHandler responseHandler = new MockResponseHandler(); + DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null); + filter.filter(request, responseHandler); - Response response = responseHandler.getResponse(); - assertNotNull(response); - assertEquals(403, response.getStatus()); - String content = responseHandler.readAll(); - assertThat(content, containsString(AuthorizationResult.DENY.getDescription())); + assertStatusCode(responseHandler, 401); + } + + @Test + public void returns_forbidden_for_credentials_rejected_by_zpe() { + AthenzAuthorizationFilter filter = createFilter(new DenyingZpe(), List.of()); + + MockResponseHandler responseHandler = new MockResponseHandler(); + DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null); + filter.filter(request, responseHandler); + + assertStatusCode(responseHandler, 403); + assertAuthorizationResult(request, Type.DENY); + } + + private static X509Certificate createDummyIdentityCertificate(AthenzIdentity identity) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal x500Name = new X500Principal("CN="+ identity.getFullName()); + Instant now = Instant.now(); + return X509CertificateBuilder + .fromKeypair(keyPair, x500Name, now, now.plus(Duration.ofDays(30)), SHA256_WITH_ECDSA, BigInteger.ONE) + .build(); + } + + private static X509Certificate createDummyRoleCertificate(AthenzRole role, AthenzIdentity identity) { + KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + X500Principal x500Name = new X500Principal("CN="+ role.domain().getName() + ":role." + role.roleName()); + Instant now = Instant.now(); + return X509CertificateBuilder + .fromKeypair(keyPair, x500Name, now, now.plus(Duration.ofDays(30)), SHA256_WITH_ECDSA, BigInteger.ONE) + .addSubjectAlternativeName(new SubjectAlternativeName(RFC822_NAME, identity.getFullName() + "@my.domain.my-identity-provider")) + .build(); } - private static DiscFilterRequest createRequest() { + private static DiscFilterRequest createRequest(ZToken roleToken, AthenzAccessToken accessToken, X509Certificate clientCert) { DiscFilterRequest request = Mockito.mock(DiscFilterRequest.class); - when(request.getHeader(HEADER_NAME)).thenReturn("v=Z1;d=domain;r=my-role;p=my-domain.my-service"); + when(request.getHeader(HEADER_NAME)).thenReturn(roleToken != null ? roleToken.getRawToken() : null); + when(request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME)).thenReturn(accessToken != null ? "Bearer " + accessToken.value() : null); when(request.getMethod()).thenReturn("GET"); when(request.getRequestURI()).thenReturn("/my/path"); when(request.getQueryString()).thenReturn(null); - when(request.getClientCertificateChain()).thenReturn(emptyList()); + when(request.getClientCertificateChain()).thenReturn(clientCert != null ? List.of(clientCert) : List.of()); return request; } - static class AllowingZpe implements Zpe { + private static AthenzAuthorizationFilter createFilter(Zpe zpe, List<EnabledCredentials.Enum> enabledCredentials) { + return new AthenzAuthorizationFilter( + new AthenzAuthorizationFilterConfig( + new AthenzAuthorizationFilterConfig.Builder() + .roleTokenHeaderName(HEADER_NAME) + .enabledCredentials(enabledCredentials)), + new StaticRequestResourceMapper(RESOURCE_NAME, ACTION), + zpe); + } + + private static void assertAuthorizationResult(DiscFilterRequest request, Type expectedResult) { + verify(request).setAttribute(RESULT_ATTRIBUTE, expectedResult.name()); + } + + private static void assertStatusCode(MockResponseHandler responseHandler, int statusCode) { + Response response = responseHandler.getResponse(); + assertThat(response, notNullValue()); + assertThat(response.getStatus(), equalTo(statusCode)); + } + + private static void assertMatchedCredentialType(DiscFilterRequest request, EnabledCredentials.Enum expectedType) { + verify(request).setAttribute(MATCHED_CREDENTIAL_TYPE_ATTRIBUTE, expectedType.name()); + } + + private static void assertRequestNotFiltered(MockResponseHandler responseHandler) { + assertThat(responseHandler.getResponse(), nullValue()); + } + + private static void assertMatchedRole(DiscFilterRequest request, AthenzRole role) { + verify(request).setAttribute(MATCHED_ROLE_ATTRIBUTE, role.roleName()); + } + + private static class AllowingZpe implements Zpe { + @Override public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) { - return AuthorizationResult.ALLOW; + return new AuthorizationResult(Type.ALLOW, ROLE); } @Override public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) { - return AuthorizationResult.ALLOW; + return new AuthorizationResult(Type.ALLOW, ROLE); + } + + @Override + public AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) { + return new AuthorizationResult(Type.ALLOW, ROLE); } } - static class DenyingZpe implements Zpe { + private static class DenyingZpe implements Zpe { @Override public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) { - return AuthorizationResult.DENY; + return new AuthorizationResult(Type.DENY); } @Override public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) { - return AuthorizationResult.DENY; + return new AuthorizationResult(Type.DENY); + } + + @Override + public AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) { + return new AuthorizationResult(Type.DENY); } } diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 1615cf7e686..508a6a84974 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -119,9 +119,16 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.util.Collection)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()" ], - "fields": [] + "fields": [ + "public java.util.List enabledCipherSuites", + "public java.util.List enabledProtocols" + ] }, "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum": { "superClass": "java.lang.Enum", @@ -174,7 +181,11 @@ "public java.lang.String certificate()", "public java.lang.String caCertificateFile()", "public java.lang.String caCertificate()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()" + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()", + "public java.util.List enabledCipherSuites()", + "public java.lang.String enabledCipherSuites(int)", + "public java.util.List enabledProtocols()", + "public java.lang.String enabledProtocols(int)" ], "fields": [] }, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index ffff63a424e..aeb08e042a1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -5,22 +5,28 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.ConnectorConfig; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.NoConnectionReuseStrategy; +import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.ssl.SSLContexts; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.security.GeneralSecurityException; +import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,6 +59,9 @@ class HealthCheckProxyHandler extends HandlerWrapper { ConnectorConfig.HealthCheckProxy proxyConfig = connector.connectorConfig().healthCheckProxy(); if (proxyConfig.enable()) { mapping.put(connector.listenPort(), createProxyTarget(proxyConfig.port(), connectors)); + log.info(String.format("Port %1$d is configured as a health check proxy for port %2$d. " + + "HTTP requests to '%3$s' on %1$d are proxied as HTTPS to %2$d.", + connector.listenPort(), proxyConfig.port(), HEALTH_CHECK_PATH)); } } return mapping; @@ -63,9 +72,9 @@ class HealthCheckProxyHandler extends HandlerWrapper { .filter(connector -> connector.listenPort() == targetPort) .findAny() .orElseThrow(() -> new IllegalArgumentException("Could not find any connector with listen port " + targetPort)); - SslContextFactory sslContextFactory = + SslContextFactory.Server sslContextFactory = Optional.ofNullable(targetConnector.getConnectionFactory(SslConnectionFactory.class)) - .map(SslConnectionFactory::getSslContextFactory) + .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); return new ProxyTarget(targetPort, sslContextFactory); } @@ -111,18 +120,23 @@ class HealthCheckProxyHandler extends HandlerWrapper { private static class ProxyTarget implements AutoCloseable { final int port; - final SslContextFactory sslContextFactory; + final SslContextFactory.Server sslContextFactory; volatile CloseableHttpClient client; - ProxyTarget(int port, SslContextFactory sslContextFactory) { + ProxyTarget(int port, SslContextFactory.Server sslContextFactory) { this.port = port; this.sslContextFactory = sslContextFactory; } CloseableHttpResponse requestStatusHtml() throws IOException { - HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH); - request.setHeader("Connection", "Close"); - return client().execute(request); + try { + HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH); + return client().execute(request); + } catch (SSLException e) { + log.log(Level.SEVERE, "SSL connection failed. Closing existing client, a new client will be created on next request", e); + close(); + throw e; + } } // Client construction must be delayed to ensure that the SslContextFactory is started before calling getSslContext(). @@ -132,11 +146,17 @@ class HealthCheckProxyHandler extends HandlerWrapper { if (client == null) { client = HttpClientBuilder.create() .disableAutomaticRetries() - .setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE) - .setSSLContext(sslContextFactory.getSslContext()) + .setMaxConnPerRoute(4) + .setSSLContext(getSslContext(sslContextFactory)) .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 .setUserAgent("health-check-proxy-client") + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout((int) Duration.ofSeconds(4).toMillis()) + .setConnectionRequestTimeout((int) Duration.ofSeconds(4).toMillis()) + .setSocketTimeout((int) Duration.ofSeconds(8).toMillis()) + .build()) .build(); } } @@ -144,11 +164,31 @@ class HealthCheckProxyHandler extends HandlerWrapper { return client; } + private SSLContext getSslContext(SslContextFactory.Server sslContextFactory) { + if (sslContextFactory.getNeedClientAuth()) { + log.info(String.format("Port %d requires client certificate. HTTPS client will use the target server connector's ssl context.", port)); + // A client certificate is only required if the server connector's ssl context factory is configured with "need-auth". + // We use the server's ssl context (truststore + keystore) if a client certificate is required. + // This will only work if the server certificate's CA is in the truststore. + return sslContextFactory.getSslContext(); + } else { + log.info(String.format( + "Port %d does not require a client certificate. HTTPS client will use a custom ssl context accepting all certificates.", port)); + // No client certificate required. The client is configured with a trust manager that accepts all certificates. + try { + return SSLContexts.custom().loadTrustMaterial(new TrustAllStrategy()).build(); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + } + @Override public void close() throws IOException { synchronized (this) { if (client != null) { client.close(); + client = null; } } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java index 48a7c246500..90848f1dfd4 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -2,14 +2,16 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.security.tls.DefaultTlsContext; -import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.AutoReloadingX509KeyManager; import com.yahoo.security.tls.TlsContext; import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; @@ -17,16 +19,21 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.security.PrivateKey; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; + /** * An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. * * @author bjorncs */ -public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider { +public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider { + private volatile AutoReloadingX509KeyManager keyManager; private final ConnectorConfig connectorConfig; public ConfiguredSslContextFactoryProvider(ConnectorConfig connectorConfig) { @@ -35,17 +42,50 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider } @Override - protected TlsContext getTlsContext(String containerId, int port) { + public SslContextFactory getInstance(String containerId, int port) { ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); if (!sslConfig.enabled()) throw new IllegalStateException(); - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig)); - List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig)); + SslContextBuilder builder = new SslContextBuilder(); + if (sslConfig.certificateFile().isBlank() || sslConfig.privateKeyFile().isBlank()) { + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig)); + List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig)); + builder.withKeyStore(privateKey, certificates); + } else { + keyManager = AutoReloadingX509KeyManager.fromPemFiles(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile())); + builder.withKeyManager(keyManager); + } List<X509Certificate> caCertificates = getCaCertificates(sslConfig) .map(X509CertificateUtils::certificateListFromPem) .orElse(List.of()); - PeerAuthentication peerAuthentication = toPeerAuthentication(sslConfig.clientAuth()); - return new DefaultTlsContext(certificates, privateKey, caCertificates, null, null, peerAuthentication); + builder.withTrustStore(caCertificates); + + SSLContext sslContext = builder.build(); + + SslContextFactory.Server factory = new SslContextFactory.Server(); + factory.setSslContext(sslContext); + + factory.setNeedClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.NEED_AUTH); + factory.setWantClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.WANT_AUTH); + + List<String> protocols = !sslConfig.enabledProtocols().isEmpty() + ? sslConfig.enabledProtocols() + : new ArrayList<>(TlsContext.getAllowedProtocols(sslContext)); + setEnabledProtocols(factory, sslContext, protocols); + + List<String> ciphers = !sslConfig.enabledCipherSuites().isEmpty() + ? sslConfig.enabledCipherSuites() + : new ArrayList<>(TlsContext.getAllowedCipherSuites(sslContext)); + setEnabledCipherSuites(factory, sslContext, ciphers); + + return factory; + } + + @Override + public void close() { + if (keyManager != null) { + keyManager.close(); + } } private static void validateConfig(ConnectorConfig.Ssl config) { @@ -64,19 +104,6 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider throw new IllegalArgumentException("Specified neither private key or private key file."); } - private static PeerAuthentication toPeerAuthentication(ConnectorConfig.Ssl.ClientAuth.Enum clientAuth) { - switch (clientAuth) { - case DISABLED: - return PeerAuthentication.DISABLED; - case NEED_AUTH: - return PeerAuthentication.NEED; - case WANT_AUTH: - return PeerAuthentication.WANT; - default: - throw new IllegalArgumentException("Unknown client auth: " + clientAuth); - } - } - private static boolean hasBoth(String a, String b) { return !a.isBlank() && !b.isBlank(); } private static boolean hasNeither(String a, String b) { return a.isBlank() && b.isBlank(); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java new file mode 100644 index 00000000000..a0172668cbb --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLContext; +import java.util.Arrays; +import java.util.List; + +/** + * @author bjorncs + */ +class SslContextFactoryUtils { + + static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContext, List<String> enabledCiphers) { + String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); + factory.setIncludeCipherSuites(enabledCiphers.toArray(String[]::new)); + factory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); + } + + static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContext, List<String> enabledProtocols) { + String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); + factory.setIncludeProtocols(enabledProtocols.toArray(String[]::new)); + factory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + } + + private static String[] createExclusionList(List<String> enabledValues, String[] supportedValues) { + return Arrays.stream(supportedValues) + .filter(supportedValue -> !enabledValues.contains(supportedValue)) + .toArray(String[]::new); + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java index e8ae13e48be..93d4f1dca3f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java @@ -8,7 +8,10 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; -import java.util.Arrays; +import java.util.List; + +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; /** * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances. @@ -31,24 +34,9 @@ public abstract class TlsContextBasedProvider extends AbstractComponent implemen sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth()); sslContextFactory.setWantClientAuth(parameters.getWantClientAuth()); - String[] enabledProtocols = parameters.getProtocols(); - sslContextFactory.setIncludeProtocols(enabledProtocols); - String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); - sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + setEnabledProtocols(sslContextFactory, sslContext, List.of(parameters.getProtocols())); + setEnabledCipherSuites(sslContextFactory, sslContext, List.of(parameters.getCipherSuites())); - String[] enabledCiphers = parameters.getCipherSuites(); - String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); - sslContextFactory.setIncludeCipherSuites(enabledCiphers); - sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); return sslContextFactory; } - - private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) { - return Arrays.stream(supportedValues) - .filter(supportedValue -> - Arrays.stream(enabledValues) - .noneMatch(enabledValue -> enabledValue.equals(supportedValue))) - .toArray(String[]::new); - } - } diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 1122b1db3a9..fe79ec2ffa3 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -81,6 +81,12 @@ ssl.caCertificate string default="" # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED +# List of enabled cipher suites. JDisc will use Vespa default if empty. +ssl.enabledCipherSuites[] string + +# List of enabled TLS protocol versions. JDisc will use Vespa default if empty. +ssl.enabledProtocols[] string + # Enforce TLS client authentication for https requests at the http layer. # Intended to be used with connectors with optional client authentication enabled. # 401 status code is returned for requests from non-authenticated clients. diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java index d4e1a15b957..c9c6d78ffba 100644 --- a/jrt/src/com/yahoo/jrt/Connection.java +++ b/jrt/src/com/yahoo/jrt/Connection.java @@ -93,7 +93,7 @@ class Connection extends Target { this.parent = parent; this.owner = owner; - this.socket = parent.transport().createCryptoSocket(channel, true); + this.socket = parent.transport().createServerCryptoSocket(channel); this.spec = null; server = true; owner.sessionInit(this); @@ -171,7 +171,7 @@ class Connection extends Target { return this; } try { - socket = parent.transport().createCryptoSocket(SocketChannel.open(spec.resolveAddress()), false); + socket = parent.transport().createClientCryptoSocket(SocketChannel.open(spec.resolveAddress()), spec); } catch (Exception e) { setLostReason(e); } diff --git a/jrt/src/com/yahoo/jrt/Connector.java b/jrt/src/com/yahoo/jrt/Connector.java index 4c83a2884bd..57fad5a163d 100644 --- a/jrt/src/com/yahoo/jrt/Connector.java +++ b/jrt/src/com/yahoo/jrt/Connector.java @@ -1,75 +1,43 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; +import com.yahoo.concurrent.ThreadFactoryFactory; -class Connector { +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; - private class Run implements Runnable { - public void run() { - try { - Connector.this.run(); - } catch (Throwable problem) { - parent.handleFailure(problem, Connector.this); - } - } - } - - private Thread thread = new Thread(new Run(), "<jrt-connector>"); - private Transport parent; - private ThreadQueue connectQueue = new ThreadQueue(); - private boolean done = false; - private boolean exit = false; +class Connector { - public Connector(Transport parent) { - this.parent = parent; - thread.setDaemon(true); - thread.start(); - } + private final ExecutorService executor = new ThreadPoolExecutor(1, 64, 1L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getDaemonThreadFactory("jrt.connector")); - public void connectLater(Connection c) { - if ( ! connectQueue.enqueue(c)) { - c.transportThread().addConnection(c); - } + private void connect(Connection conn) { + conn.transportThread().addConnection(conn.connect()); } - private void run() { + public void connectLater(Connection conn) { try { - while (true) { - Connection conn = (Connection) connectQueue.dequeue(); - conn.transportThread().addConnection(conn.connect()); - } - } catch (EndOfQueueException e) {} - synchronized (this) { - done = true; - notifyAll(); - while (!exit) { - try { wait(); } catch (InterruptedException x) {} - } + executor.execute(() -> connect(conn)); + } catch (RejectedExecutionException e) { + conn.transportThread().addConnection(conn); } } public Connector shutdown() { - connectQueue.close(); - return this; - } - - public synchronized void waitDone() { - while (!done) { - try { wait(); } catch (InterruptedException x) {} - } - } - - public synchronized Connector exit() { - exit = true; - notifyAll(); + executor.shutdown(); return this; } public void join() { while (true) { try { - thread.join(); - return; + if (executor.awaitTermination(60, TimeUnit.SECONDS)) { + return; + } } catch (InterruptedException e) {} } } diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 8812264a3f1..6d1955d7f66 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -18,7 +18,8 @@ import java.nio.channels.SocketChannel; * encryption. **/ public interface CryptoEngine extends AutoCloseable { - CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer); + CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec); + CryptoSocket createServerCryptoSocket(SocketChannel channel); static CryptoEngine createDefault() { if (!TransportSecurityUtils.isTransportSecurityEnabled()) { return new NullCryptoEngine(); diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java index 801f2075c4e..18549df6f2c 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java @@ -21,17 +21,20 @@ public class MaybeTlsCryptoEngine implements CryptoEngine { } @Override - public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - if (isServer) { - return new MaybeTlsCryptoSocket(channel, tlsEngine, isServer); - } else if (useTlsWhenClient) { - return tlsEngine.createCryptoSocket(channel, false); + public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + if (useTlsWhenClient) { + return tlsEngine.createClientCryptoSocket(channel, spec); } else { - return new NullCryptoSocket(channel, isServer); + return new NullCryptoSocket(channel, false); } } @Override + public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new MaybeTlsCryptoSocket(channel, tlsEngine); + } + + @Override public String toString() { return "MaybeTlsCryptoEngine(useTlsWhenClient:" + useTlsWhenClient + ")"; } @Override diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java index 5c4510665e7..60b7f342c9c 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java @@ -61,8 +61,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { private TlsCryptoEngine factory; private Buffer buffer; - MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) { - super(channel, isServer); + MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) { + super(channel, true); this.factory = factory; this.buffer = new Buffer(4096); } @@ -81,7 +81,7 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { data[i] = src.get(i); } if (looksLikeTlsToMe(data)) { - TlsCryptoSocket tlsSocket = factory.createCryptoSocket(channel(), true); + TlsCryptoSocket tlsSocket = factory.createServerCryptoSocket(channel()); tlsSocket.injectReadData(buffer); socket = tlsSocket; return socket.handshake(); @@ -117,8 +117,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { } } - public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) { - this.socket = new MyCryptoSocket(channel, factory, isServer); + public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) { + this.socket = new MyCryptoSocket(channel, factory); } @Override public SocketChannel channel() { return socket.channel(); } diff --git a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java index b5a53accf92..b97ec17a5dc 100644 --- a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java @@ -9,7 +9,10 @@ import java.nio.channels.SocketChannel; * CryptoEngine implementation that performs no encryption. **/ public class NullCryptoEngine implements CryptoEngine { - @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return new NullCryptoSocket(channel, isServer); + @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return new NullCryptoSocket(channel, false); + } + @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new NullCryptoSocket(channel, true); } } diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java index 84fbb7d4f01..7474220d4e7 100644 --- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java @@ -20,9 +20,16 @@ public class TlsCryptoEngine implements CryptoEngine { } @Override - public TlsCryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { + public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { SSLEngine sslEngine = tlsContext.createSslEngine(); - sslEngine.setUseClientMode(!isServer); + sslEngine.setUseClientMode(true); + return new TlsCryptoSocket(channel, sslEngine); + } + + @Override + public TlsCryptoSocket createServerCryptoSocket(SocketChannel channel) { + SSLEngine sslEngine = tlsContext.createSslEngine(); + sslEngine.setUseClientMode(false); return new TlsCryptoSocket(channel, sslEngine); } diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index f4eb1acd096..6f5a381fd6b 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -46,7 +46,7 @@ public class Transport { this.fatalHandler = fatalHandler; // NB: this must be set first } this.cryptoEngine = cryptoEngine; - connector = new Connector(this); + connector = new Connector(); worker = new Worker(this); runCnt = new AtomicInteger(numThreads); for (int i = 0; i < numThreads; ++i) { @@ -68,14 +68,26 @@ public class Transport { } /** - * Use the underlying CryptoEngine to create a CryptoSocket. + * Use the underlying CryptoEngine to create a CryptoSocket for + * the client side of a connection. * * @return CryptoSocket handling appropriate encryption * @param channel low-level socket channel to be wrapped by the CryptoSocket - * @param isServer flag indicating which end of the connection we are + * @param spec who we are connecting to, for hostname validation **/ - CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return cryptoEngine.createCryptoSocket(channel, isServer); + CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return cryptoEngine.createClientCryptoSocket(channel, spec); + } + + /** + * Use the underlying CryptoEngine to create a CryptoSocket for + * the server side of a connection. + * + * @return CryptoSocket handling appropriate encryption + * @param channel low-level socket channel to be wrapped by the CryptoSocket + **/ + CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return cryptoEngine.createServerCryptoSocket(channel); } /** @@ -162,7 +174,7 @@ public class Transport { * @return this object, to enable chaining with join **/ public Transport shutdown() { - connector.shutdown().waitDone(); + connector.shutdown().join(); for (TransportThread thread: threads) { thread.shutdown(); } @@ -181,7 +193,6 @@ public class Transport { void notifyDone(TransportThread self) { if (runCnt.decrementAndGet() == 0) { worker.shutdown().join(); - connector.exit().join(); try { cryptoEngine.close(); } catch (Exception e) {} } } diff --git a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java index 4ba6d00faa4..d720ca4dc26 100644 --- a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java @@ -11,7 +11,10 @@ import java.nio.channels.SocketChannel; * from TLS. **/ public class XorCryptoEngine implements CryptoEngine { - @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { - return new XorCryptoSocket(channel, isServer); + @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { + return new XorCryptoSocket(channel, false); + } + @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) { + return new XorCryptoSocket(channel, true); } } diff --git a/juniper/src/vespa/juniper/expcache.h b/juniper/src/vespa/juniper/expcache.h index 6939785914a..beaa491148c 100644 --- a/juniper/src/vespa/juniper/expcache.h +++ b/juniper/src/vespa/juniper/expcache.h @@ -2,6 +2,7 @@ #pragma once #include "simplemap.h" +#include <cstdint> class MatchObject; diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index addf9125508..bf98bbd75ef 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -46,10 +46,11 @@ bool fixDir(const vespalib::string &path) { } vespalib::string -cfFilePath(const vespalib::string &parent) { +cfFilePath(const vespalib::string &parent, const vespalib::string &filename) { vespalib::string path = parent + "/etc/system/local"; fixDir(path); - path += "/deploymentclient.conf"; + path += "/"; + path += filename; return path; } @@ -61,7 +62,7 @@ CfHandler::doConfigure() std::unique_ptr<LogforwarderConfig> cfg(_handle->getConfig()); const LogforwarderConfig& config(*cfg); - vespalib::string path = cfFilePath(config.splunkHome); + vespalib::string path = cfFilePath(config.splunkHome, "deploymentclient.conf"); vespalib::string tmpPath = path + ".new"; FILE *fp = fopen(tmpPath.c_str(), "w"); if (fp == NULL) return; @@ -76,6 +77,22 @@ CfHandler::doConfigure() fclose(fp); rename(tmpPath.c_str(), path.c_str()); + if (getenv("VESPA_HOSTNAME") != NULL && + getenv("VESPA_TENANT") != NULL && + getenv("VESPA_APPLICATION")!= NULL && + getenv("VESPA_INSTANCE") != NULL ) + { + path = cfFilePath(config.splunkHome, "inputs.conf"); + tmpPath = path + ".new"; + fp = fopen(tmpPath.c_str(), "w"); + if (fp != NULL) { + fprintf(fp, "[default]\n"); + fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fclose(fp); + rename(tmpPath.c_str(), path.c_str()); + } + } if (config.clientName.size() == 0 || config.deploymentServer.size() == 0) { diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index 9f55e218140..913cdb78327 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000" +addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index 41fedb8e8c4..7587159165d 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -6,5 +6,7 @@ install_config_definition(src/main/resources/configdefinitions/consumers.def ai. install_config_definition(src/main/resources/configdefinitions/monitoring.def ai.vespa.metricsproxy.core.monitoring.def) install_config_definition(src/main/resources/configdefinitions/metrics-nodes.def ai.vespa.metricsproxy.http.application.metrics-nodes.def) install_config_definition(src/main/resources/configdefinitions/node-dimensions.def ai.vespa.metricsproxy.metric.dimensions.node-dimensions.def) +install_config_definition(src/main/resources/configdefinitions/node-info.def ai.vespa.metricsproxy.http.metrics.node-info.def) install_config_definition(src/main/resources/configdefinitions/rpc-connector.def ai.vespa.metricsproxy.rpc.rpc-connector.def) install_config_definition(src/main/resources/configdefinitions/vespa-services.def ai.vespa.metricsproxy.service.vespa-services.def) +install_config_definition(src/main/resources/configdefinitions/telegraf.def ai.vespa.metricsproxy.telegraf.telegraf.def) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 4c4015220bc..53a05ef88f0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -79,6 +79,16 @@ public class MetricsManager { * @return Metrics for all matching services. */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { + return getMetricsAsBuilders(services, startTime).stream() + .map(MetricsPacket.Builder::build) + .collect(Collectors.toList()); + } + + /** + * Returns the metrics for the given services, in mutable state for further processing. + * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary. + */ + public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) { if (services.isEmpty()) return Collections.emptyList(); log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); @@ -99,7 +109,6 @@ public class MetricsManager { .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions())) .map(builder -> builder.putDimensionsIfAbsent(extraDimensions)) .map(builder -> adjustTimestamp(builder, startTime)) - .map(MetricsPacket.Builder::build) .collect(Collectors.toList()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index 1b03d3b01f9..c04dca465a1 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; @@ -40,7 +41,7 @@ public class VespaMetrics { public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype"); - public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance"); + public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId(INTERNAL_SERVICE_ID); private final MetricsConsumers metricsConsumers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 7386305ad34..51bdae1aab3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -47,6 +47,16 @@ public class ValuesFetcher { .collect(Collectors.toList()); } + public List<MetricsPacket.Builder> fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { + ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); + + return metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now()) + .stream() + .filter(builder -> builder.hasConsumer(consumer)) + .collect(Collectors.toList()); + } + + public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException { return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index ce2e383f0d2..d9303e80dcd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -3,17 +3,17 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.http.ErrorResponse; -import ai.vespa.metricsproxy.http.HttpHandlerBase; -import ai.vespa.metricsproxy.http.JsonResponse; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; import java.net.URI; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,6 +23,7 @@ import java.util.logging.Level; import static ai.vespa.metricsproxy.http.ValuesFetcher.getConsumerOrDefault; import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; +import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.OK; import static java.util.stream.Collectors.toList; @@ -35,7 +36,9 @@ import static java.util.stream.Collectors.toList; public class ApplicationMetricsHandler extends HttpHandlerBase { public static final String V1_PATH = "/applicationmetrics/v1"; - static final String VALUES_PATH = V1_PATH + "/values"; + public static final String VALUES_PATH = V1_PATH + "/values"; + + private static final int MAX_DIMENSIONS = 10; private final ApplicationMetricsRetriever metricsRetriever; private final MetricsConsumers metricsConsumers; @@ -60,7 +63,10 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { try { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); var buildersByNode = metricsRetriever.getMetrics(consumer); - var metricsByNode = processAndBuild(buildersByNode); + var metricsByNode = processAndBuild(buildersByNode, + new ServiceIdDimensionProcessor(), + new ClusterIdDimensionProcessor(), + new PublicDimensionsProcessor(MAX_DIMENSIONS)); return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); } catch (Exception e) { @@ -69,8 +75,8 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { } } - private Map<Node, List<MetricsPacket>> processAndBuild(Map<Node, List<MetricsPacket.Builder>> buildersByNode, - MetricsProcessor... processors) { + private static Map<Node, List<MetricsPacket>> processAndBuild(Map<Node, List<MetricsPacket.Builder>> buildersByNode, + MetricsProcessor... processors) { var metricsByNode = new HashMap<Node, List<MetricsPacket>>(); buildersByNode.forEach((node, builders) -> { @@ -84,14 +90,4 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { return metricsByNode; } - private MetricsPacket.Builder applyProcessors(MetricsPacket.Builder builder, MetricsProcessor... processors) { - Arrays.stream(processors).forEach(processor -> processor.process(builder)); - return builder; - } - - interface MetricsProcessor { - // Processes the metrics packet builder in-place. - void process(MetricsPacket.Builder builder); - } - } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java new file mode 100644 index 00000000000..292c6da3de2 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java @@ -0,0 +1,39 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.CLUSTER_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_TYPE; +import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; + +/** + * Replaces the current cluster ID dimension value with "clustertype/clusterid". + * + * @author gjoranv + */ +public class ClusterIdDimensionProcessor implements MetricsProcessor { + + @Override + public void process(MetricsPacket.Builder builder) { + String clusterType = emptyIfNull(builder.getDimensionValue(toDimensionId(INTERNAL_CLUSTER_TYPE))); + String clusterId = emptyIfNull(builder.getDimensionValue(toDimensionId(INTERNAL_CLUSTER_ID))); + + String newClusterId; + if (! clusterType.isEmpty() && ! clusterId.isEmpty()) + newClusterId = clusterType + "/" + clusterId; + else if (! clusterType.isEmpty()) + newClusterId = clusterType; + else if (! clusterId.isEmpty()) + newClusterId = clusterId; + else + return; // Both type and id were null or empty + + builder.putDimension(toDimensionId(CLUSTER_ID), newClusterId); + } + + private String emptyIfNull(String s) { + return s == null ? "" : s; + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index c8a8e65be5d..c439a037774 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,7 +25,7 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null configId is not allowed"); + Objects.requireNonNull(role, "Null role is not allowed"); Objects.requireNonNull(hostname, "Null hostname is not allowed"); Objects.requireNonNull(path, "Null path is not allowed"); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java new file mode 100644 index 00000000000..4d1d57644b5 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java @@ -0,0 +1,69 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Ensures that only whitelisted dimensions are retained in the given metrics packet, and that + * there are no more dimensions than the given maximum number. + * + * @author gjoranv + */ +public class PublicDimensionsProcessor implements MetricsProcessor { + + private final int maxDimensions; + private Set<DimensionId> publicDimensions = getPublicDimensions(); + + public PublicDimensionsProcessor(int maxDimensions) { + int numCommonDimensions = PublicDimensions.commonDimensions.size(); + if (numCommonDimensions > maxDimensions) { + throw new IllegalArgumentException(String.format( + ("The maximum number of dimensions (%d) cannot be smaller than the number of " + + "common metrics dimensions (%d)."), maxDimensions, numCommonDimensions)); + } + this.maxDimensions = maxDimensions; + } + + @Override + public void process(MetricsPacket.Builder builder) { + Set<DimensionId> dimensionsToRetain = builder.getDimensionIds(); + dimensionsToRetain.retainAll(publicDimensions); + + if (dimensionsToRetain.size() > maxDimensions) { + for (var metricDim : getMetricDimensions()) { + dimensionsToRetain.remove(metricDim); + if (dimensionsToRetain.size() <= maxDimensions) break; + } + } + + builder.retainDimensions(dimensionsToRetain); + + // Extra safeguard, to make sure we don't exceed the limit of some metric systems. + if (builder.getDimensionIds().size() > maxDimensions) { + throw new IllegalStateException(String.format( + "Metrics packet is only allowed to have %d dimensions, but has: %s", maxDimensions, builder.getDimensionIds())); + } + } + + static Set<DimensionId> getPublicDimensions() { + return toDimensionIds(PublicDimensions.publicDimensions); + } + + static Set<DimensionId> getMetricDimensions() { + return toDimensionIds(PublicDimensions.metricDimensions); + } + + static Set<DimensionId> toDimensionIds(Collection<String> dimensionNames) { + return dimensionNames.stream() + .map(DimensionId::toDimensionId) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + } +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java new file mode 100644 index 00000000000..45f211a5704 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java @@ -0,0 +1,24 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID; +import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; + +/** + * Copies the value of the internally used 'instance' dimension to the more aptly named 'serviceId'. + * + * @author gjoranv + */ +public class ServiceIdDimensionProcessor implements MetricsProcessor { + + @Override + public void process(MetricsPacket.Builder builder) { + String serviceIdValue = builder.getDimensionValue(toDimensionId(INTERNAL_SERVICE_ID)); + if (serviceIdValue != null) + builder.putDimension(toDimensionId(SERVICE_ID), serviceIdValue); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java index 28a24c5dc25..bf4d7b55989 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java @@ -3,13 +3,13 @@ package ai.vespa.metricsproxy.http.metrics; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.http.ErrorResponse; -import ai.vespa.metricsproxy.http.HttpHandlerBase; -import ai.vespa.metricsproxy.http.JsonResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java new file mode 100644 index 00000000000..71d7857e48a --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -0,0 +1,92 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor; +import ai.vespa.metricsproxy.http.application.Node; +import ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor; +import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; +import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toList; + +/** + * Http handler for the metrics/v2 rest api. + * + * @author gjoranv + */ +public class MetricsV2Handler extends HttpHandlerBase { + + public static final String V2_PATH = "/metrics/v2"; + public static final String VALUES_PATH = V2_PATH + "/values"; + private static final int MAX_DIMENSIONS = 10; + + private final ValuesFetcher valuesFetcher; + private final NodeInfoConfig nodeInfoConfig; + + @Inject + public MetricsV2Handler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + NodeInfoConfig nodeInfoConfig) { + super(executor); + this.nodeInfoConfig = nodeInfoConfig; + valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + } + + @Override + public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List<MetricsPacket.Builder> builders = valuesFetcher.fetchMetricsAsBuilders(consumer); + List<MetricsPacket> metrics = processAndBuild(builders, + new ServiceIdDimensionProcessor(), + new ClusterIdDimensionProcessor(), + new PublicDimensionsProcessor(MAX_DIMENSIONS)); + + Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, ""); + Map<Node, List<MetricsPacket>> metricsByNode = singletonMap(localNode, metrics); + return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); + } catch (Exception e) { + log.log(Level.WARNING, "Got exception when rendering metrics:", e); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static List<MetricsPacket> processAndBuild(List<MetricsPacket.Builder> builders, + MetricsProcessor... processors) { + return builders.stream() + .map(builder -> applyProcessors(builder, processors)) + .map(MetricsPacket.Builder::build) + .collect(toList()); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java index 5f3723df94d..eeabb1e03ac 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java @@ -3,12 +3,12 @@ package ai.vespa.metricsproxy.http.prometheus; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.http.HttpHandlerBase; import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.HttpHandlerBase; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java index 38011d089d4..2b22ea101d8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -4,17 +4,17 @@ package ai.vespa.metricsproxy.http.yamas; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.http.ValuesFetcher; -import ai.vespa.metricsproxy.node.NodeMetricGatherer; -import ai.vespa.metricsproxy.http.ErrorResponse; -import ai.vespa.metricsproxy.http.HttpHandlerBase; -import ai.vespa.metricsproxy.http.JsonResponse; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException; import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil; +import ai.vespa.metricsproxy.node.NodeMetricGatherer; import ai.vespa.metricsproxy.service.VespaServices; import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java new file mode 100644 index 00000000000..8d525310d6f --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java @@ -0,0 +1,77 @@ +package ai.vespa.metricsproxy.metric.dimensions; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * The names of all dimensions that are publicly available, in addition to some dimensions that + * are used in the process of composing these public dimensions. + * + * 'INTERNAL' in this context means non-public. + * + * @author gjoranv + */ +public final class PublicDimensions { + private PublicDimensions() { } + + public static final String APPLICATION_ID = "applicationId"; // <tenant.app.instance> + public static final String ZONE = "zone"; + + // The public CLUSTER_ID dimension value is composed from the two non-public dimensions. + // Node-specific. + public static final String INTERNAL_CLUSTER_TYPE = "clustertype"; + public static final String INTERNAL_CLUSTER_ID = "clusterid"; + public static final String CLUSTER_ID = "clusterId"; + + // Internal name (instance) is confusing, so renamed to 'serviceId' for public use. + // This is added by the metrics-proxy. + public static final String INTERNAL_SERVICE_ID = "instance"; + public static final String SERVICE_ID = "serviceId"; + + // From host-admin, currently (Jan 2020) only included for 'vespa.node' metrics + public static final String HOSTNAME = "host"; + + + /** Metric specific dimensions **/ + public static final String API = "api"; // feed + public static final String CHAIN = "chain"; // query + public static final String DOCUMENT_TYPE = "documenttype"; // content + public static final String ENDPOINT = "endpoint"; // query + public static final String GC_NAME = "gcName"; // container + public static final String HTTP_METHOD = "httpMethod"; // container + public static final String OPERATION = "operation"; // feed + public static final String RANK_PROFILE = "rankProfile"; // content + public static final String REASON = "reason"; // query (degraded etc.) + public static final String STATUS = "status"; // feed + + + // Dimensions that are valid (but not necessarily used) for all metrics. + public static List<String> commonDimensions = + List.of(APPLICATION_ID, + CLUSTER_ID, + HOSTNAME, + SERVICE_ID, + ZONE); + + // Dimensions that are only used for a subset of metrics. + public static List<String> metricDimensions = + List.of(API, + CHAIN, + DOCUMENT_TYPE, + ENDPOINT, + GC_NAME, + HTTP_METHOD, + OPERATION, + RANK_PROFILE, + REASON, + STATUS); + + + /** + * All public dimensions, common dimensions first, then dimensions for individual metrics + */ + public static final List<String> publicDimensions = Stream.concat(commonDimensions.stream(), metricDimensions.stream()) + .collect(Collectors.toList()); + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index f9cc17f94ea..8d5a1f50918 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -158,11 +158,31 @@ public class MetricsPacket { return this; } + /** + * Returns a modifiable copy of the dimension IDs of this builder, usually for use with {@link #retainDimensions(Collection)}. + */ + public Set<DimensionId> getDimensionIds() { + return new LinkedHashSet<>(dimensions.keySet()); + } + + public String getDimensionValue(DimensionId id) { + return dimensions.get(id); + } + + public Builder retainDimensions(Collection<DimensionId> idsToRetain) { + dimensions.keySet().retainAll(idsToRetain); + return this; + } + public Builder addConsumers(Set<ConsumerId> extraConsumers) { if (extraConsumers != null) consumers.addAll(extraConsumers); return this; } + public boolean hasConsumer(ConsumerId id) { + return consumers.contains(id); + } + public MetricsPacket build() { return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java new file mode 100644 index 00000000000..7860825e075 --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java @@ -0,0 +1,31 @@ +package ai.vespa.metricsproxy.metric.model.processing; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; + +import java.util.Arrays; + +/** + * Interface for classes that make amendments to a metrics packet builder. + * Includes a utility method to apply a list of processors to a metrics packet. + * + * @author gjoranv + */ +public interface MetricsProcessor { + + /** + * Processes the metrics packet builder in-place. + */ + void process(MetricsPacket.Builder builder); + + + /** + * Helper method to apply a list of processors to a metrics packet builder. + * Returns the metrics packet builder (which has been processed in-place) for + * convenient use in stream processing. + */ + static MetricsPacket.Builder applyProcessors(MetricsPacket.Builder builder, MetricsProcessor... processors) { + Arrays.stream(processors).forEach(processor -> processor.process(builder)); + return builder; + } + +} diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-info.def b/metrics-proxy/src/main/resources/configdefinitions/node-info.def new file mode 100644 index 00000000000..e66433a96d0 --- /dev/null +++ b/metrics-proxy/src/main/resources/configdefinitions/node-info.def @@ -0,0 +1,5 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.metricsproxy.http.metrics + +role string +hostname string diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def new file mode 100644 index 00000000000..f3b5db35d52 --- /dev/null +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -0,0 +1,20 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.metricsproxy.telegraf + +# Metrics pull/push interval +intervalSeconds int default=60 + + +# The consumer to get metrics for +vespa.consumer string default="default" + + +cloudWatch[].region string default="us-east-1" +cloudWatch[].namespace string + +# Only valid and required for hosted Vespa +cloudWatch[].accessKeyName string default="" +cloudWatch[].secretKeyName string default="" + +# Only valid and optional for self-hosted Vespa +cloudWatch[].profile string default="" diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java index 77c3a719cd9..d776368687d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java @@ -24,6 +24,7 @@ import java.util.List; import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; /** @@ -61,11 +62,12 @@ public class HttpHandlerTestBase { } protected static MetricsConsumers getMetricsConsumers() { + // Must use a whitelisted dimension to avoid it being removed for the MetricsV2Handler var defaultConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("default-val"); + .key(REASON).value("default-val"); var customConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("custom-val"); + .key(REASON).value("custom-val"); return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new ConsumersConfig.Consumer.Builder() diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index 074b7877430..155bbf094a1 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -3,6 +3,7 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; @@ -19,6 +20,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.Executors; import static ai.vespa.metricsproxy.TestUtil.getFileContents; @@ -34,6 +36,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options import static com.yahoo.collections.CollectionUtil.first; import static java.util.stream.Collectors.toList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -138,6 +141,22 @@ public class ApplicationMetricsHandlerTest { } @Test + public void metrics_processors_are_applied() { + GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); + + GenericService searchnode = jsonModel.nodes.get(0).services.get(0); + Map<String, String> dimensions = searchnode.metrics.get(0).dimensions; + assertEquals(6, dimensions.size()); + assertEquals("music.default", dimensions.get(PublicDimensions.APPLICATION_ID)); + assertEquals("container/default", dimensions.get(PublicDimensions.CLUSTER_ID)); + assertEquals("us-west", dimensions.get(PublicDimensions.ZONE)); + assertEquals("search/", dimensions.get(PublicDimensions.API)); + assertEquals("music", dimensions.get(PublicDimensions.DOCUMENT_TYPE)); + assertEquals("qrserver0", dimensions.get(PublicDimensions.SERVICE_ID)); + assertFalse(dimensions.containsKey("non-public")); + } + + @Test public void consumer_is_propagated_in_uri_to_retriever() { GenericApplicationModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER); GenericJsonModel nodeModel = jsonModel.nodes.get(0); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java new file mode 100644 index 00000000000..0f4c6a885ec --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java @@ -0,0 +1,63 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.CLUSTER_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_TYPE; +import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; +import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author gjoranv + */ +public class ClusterIdDimensionProcessorTest { + private static final DimensionId NEW_ID_DIMENSION = toDimensionId(CLUSTER_ID); + + @Test + public void cluster_id_is_replaced_with_type_and_id() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + builder.putDimension(toDimensionId(INTERNAL_CLUSTER_TYPE), "type") ; + builder.putDimension(toDimensionId(INTERNAL_CLUSTER_ID), "id") ; + + assertEquals("type/id", newClusterId(builder)); + } + + @Test + public void cluster_id_is_type_when_id_is_null() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + builder.putDimension(toDimensionId(INTERNAL_CLUSTER_TYPE), "type") ; + + assertEquals(newClusterId(builder), "type"); + } + + @Test + public void cluster_id_is_id_when_type_is_null() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + builder.putDimension(toDimensionId(INTERNAL_CLUSTER_ID), "id") ; + + assertEquals(newClusterId(builder), "id"); + } + + @Test + public void cluster_id_is_not_added_when_both_type_and_id_are_null() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + + var processor = new ClusterIdDimensionProcessor(); + processor.process(builder); + + assertFalse(builder.getDimensionIds().contains(NEW_ID_DIMENSION)); + } + + private String newClusterId(MetricsPacket.Builder builder) { + var processor = new ClusterIdDimensionProcessor(); + processor.process(builder); + + return builder.getDimensionValue(NEW_ID_DIMENSION); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java new file mode 100644 index 00000000000..c6f47d70a1f --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java @@ -0,0 +1,59 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import static ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor.getPublicDimensions; +import static ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor.toDimensionIds; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.APPLICATION_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.commonDimensions; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.publicDimensions; +import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; +import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public class PublicDimensionsProcessorTest { + + @Test + public void non_public_dimensions_are_removed() { + var builder = new MetricsPacket.Builder(toServiceId("foo")) + .putDimension(toDimensionId("a"), ""); + + var processor = new PublicDimensionsProcessor(10); + processor.process(builder); + assertTrue(builder.getDimensionIds().isEmpty()); + } + + @Test + public void public_dimensions_are_retained() { + var builder = new MetricsPacket.Builder(toServiceId("foo")) + .putDimension(toDimensionId(APPLICATION_ID), "app"); + + var processor = new PublicDimensionsProcessor(10); + processor.process(builder); + assertEquals(1, builder.getDimensionIds().size()); + assertEquals(toDimensionId(APPLICATION_ID), builder.getDimensionIds().iterator().next()); + } + + @Test + public void common_dimensions_have_priority_when_there_are_too_many() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + getPublicDimensions() + .forEach(dimensionId -> builder.putDimension(dimensionId, dimensionId.id)); + assertEquals(publicDimensions.size(), builder.getDimensionIds().size()); + + var processor = new PublicDimensionsProcessor(commonDimensions.size()); + processor.process(builder); + + var includedDimensions = builder.getDimensionIds(); + assertEquals(commonDimensions.size(), includedDimensions.size()); + + toDimensionIds(commonDimensions).forEach(commonDimension -> + assertTrue(includedDimensions.contains(commonDimension))); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java new file mode 100644 index 00000000000..3237abdbfa3 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java @@ -0,0 +1,43 @@ +package ai.vespa.metricsproxy.http.application; + +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import org.junit.Test; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID; +import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; +import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public class ServiceIdDimensionProcessorTest { + private static final DimensionId NEW_ID_DIMENSION = toDimensionId(SERVICE_ID); + + @Test + public void new_service_id_is_added_when_internal_service_id_exists() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + builder.putDimension(toDimensionId(INTERNAL_SERVICE_ID), "service"); + + var processor = new ServiceIdDimensionProcessor(); + processor.process(builder); + + assertTrue(builder.getDimensionIds().contains(NEW_ID_DIMENSION)); + assertEquals("service", builder.getDimensionValue(NEW_ID_DIMENSION)); + } + + @Test + public void new_service_id_is_not_added_when_internal_service_id_is_null() { + var builder = new MetricsPacket.Builder(toServiceId("foo")); + + var processor = new ServiceIdDimensionProcessor(); + processor.process(builder); + + assertFalse(builder.getDimensionIds().contains(NEW_ID_DIMENSION)); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java new file mode 100644 index 00000000000..1c5ce695155 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java @@ -0,0 +1,196 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.http.HttpHandlerTestBase; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; +import ai.vespa.metricsproxy.metric.model.json.GenericService; +import ai.vespa.metricsproxy.service.DownService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID; +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; +import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; +import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase { + + static String rootUri; + static String valuesUri; + + Class<MODEL> modelClass; + + abstract GenericJsonModel getGenericJsonModel(MODEL model); + + private MODEL getResponseAsJsonModel(String consumer) { + String response = testDriver.sendRequest(valuesUri + "?consumer=" + consumer).readAll(); + try { + return createObjectMapper().readValue(response, modelClass); + } catch (IOException e) { + fail("Failed to create json model: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private GenericJsonModel getResponseAsGenericJsonModel(String consumer) { + return getGenericJsonModel(getResponseAsJsonModel(consumer)); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(rootUri + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + } + + @Test + public void root_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(rootUri).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUrl = resources.getJSONObject(0); + assertEquals(valuesUri, valuesUrl.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() throws Exception { + String response = testDriver.sendRequest(valuesUri).readAll(); + ObjectMapper mapper = createObjectMapper(); + var jsonModel = mapper.readValue(response, modelClass); + System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); + } + + @Test + public void no_explicit_consumer_gives_the_default_consumer() { + String responseDefaultConsumer = testDriver.sendRequest(valuesUri + "?consumer=default").readAll(); + String responseNoConsumer = testDriver.sendRequest(valuesUri).readAll(); + assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); + } + + @Test + public void unknown_consumer_gives_the_default_consumer() { + String response = testDriver.sendRequest(valuesUri).readAll(); + String responseUnknownConsumer = testDriver.sendRequest(valuesUri + "?consumer=not_defined").readAll(); + assertEqualsExceptTimestamps(response, responseUnknownConsumer); + } + + private void assertEqualsExceptTimestamps(String s1, String s2) { + assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); + } + + private String replaceTimestamps(String s) { + return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + } + + @Test + public void response_contains_node_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertNotNull(jsonModel.node); + assertEquals(1, jsonModel.node.metrics.size()); + assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); + } + + @Test + public void response_contains_service_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy1Metrics.dimensions.get(REASON)); + } + + @Test + public void custom_consumer_gets_only_its_whitelisted_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(CUSTOM_CONSUMER); + + assertNotNull(jsonModel.node); + // TODO: see comment in ExternalMetrics.setExtraMetrics + // assertEquals(0, jsonModel.node.metrics.size()); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals("custom-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals("custom-val", dummy1Metrics.dimensions.get(REASON)); + } + + private static GenericMetrics getMetricsForService(String serviceInstance, GenericService service) { + for (var metrics : service.metrics) { + if (getServiceIdDimension(metrics).equals(serviceInstance)) + return metrics; + } + fail("Could not find metrics for service instance " + serviceInstance); + throw new RuntimeException(); + } + + @Test + public void all_timestamps_are_equal_and_non_zero() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + Long nodeTimestamp = jsonModel.node.timestamp; + assertNotEquals(0L, (long) nodeTimestamp); + for (var service : jsonModel.services) + assertEquals(nodeTimestamp, service.timestamp); + } + + @Test + public void all_consumers_get_health_from_service_that_is_down() { + assertDownServiceHealth(DEFAULT_CONSUMER); + assertDownServiceHealth(CUSTOM_CONSUMER); + } + + private void assertDownServiceHealth(String consumer) { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(consumer); + + GenericService downService = jsonModel.services.get(1); + assertEquals(DOWN.status, downService.status.code); + assertEquals("No response", downService.status.description); + + // Service should output metric dimensions, even without metrics, because they contain important info about the service. + assertEquals(1, downService.metrics.size()); + assertEquals(0, downService.metrics.get(0).values.size()); + assertFalse(downService.metrics.get(0).dimensions.isEmpty()); + assertEquals(DownService.NAME, getServiceIdDimension(downService.metrics.get(0))); + } + + private static String getServiceIdDimension(GenericMetrics metrics) { + var instanceDimension = metrics.dimensions.get(INTERNAL_SERVICE_ID); + return instanceDimension != null ? instanceDimension : metrics.dimensions.get(SERVICE_ID); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java index 22f61114622..fe823466f7b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java @@ -1,46 +1,30 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.metrics; -import ai.vespa.metricsproxy.http.HttpHandlerTestBase; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; -import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; -import ai.vespa.metricsproxy.metric.model.json.GenericService; -import ai.vespa.metricsproxy.service.DownService; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONObject; +import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import java.io.IOException; import java.util.concurrent.Executors; -import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.V1_PATH; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.VALUES_PATH; -import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; -import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; -import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author gjoranv */ @SuppressWarnings("UnstableApiUsage") -public class MetricsV1HandlerTest extends HttpHandlerTestBase { +public class MetricsV1HandlerTest extends MetricsHandlerTestBase<GenericJsonModel> { private static final String V1_URI = URI_BASE + V1_PATH; private static final String VALUES_URI = URI_BASE + VALUES_PATH; + @BeforeClass public static void setup() { + rootUri = V1_URI; + valuesUri = VALUES_URI; var handler = new MetricsV1Handler(Executors.newSingleThreadExecutor(), getMetricsManager(), vespaServices, @@ -48,149 +32,14 @@ public class MetricsV1HandlerTest extends HttpHandlerTestBase { testDriver = new RequestHandlerTestDriver(handler); } - private GenericJsonModel getResponseAsJsonModel(String consumer) { - String response = testDriver.sendRequest(VALUES_URI + "?consumer=" + consumer).readAll(); - try { - return createObjectMapper().readValue(response, GenericJsonModel.class); - } catch (IOException e) { - fail("Failed to create json model: " + e.getMessage()); - throw new RuntimeException(e); - } - } - - @Test - public void v1_response_contains_values_uri() throws Exception { - String response = testDriver.sendRequest(V1_URI).readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("resources")); - - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); - - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUrl.getString("url")); - } - - @Ignore - @Test - public void visually_inspect_values_response() throws Exception { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - ObjectMapper mapper = createObjectMapper(); - var jsonModel = mapper.readValue(response, GenericJsonModel.class); - System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); - } - - @Test - public void no_explicit_consumer_gives_the_default_consumer() { - String responseDefaultConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=default").readAll(); - String responseNoConsumer = testDriver.sendRequest(VALUES_URI).readAll(); - assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); - } - - @Test - public void unknown_consumer_gives_the_default_consumer() { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - String responseUnknownConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=not_defined").readAll(); - assertEqualsExceptTimestamps(response, responseUnknownConsumer); - } - - private void assertEqualsExceptTimestamps(String s1, String s2) { - assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); - } - - @Test - public void response_contains_node_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertNotNull(jsonModel.node); - assertEquals(1, jsonModel.node.metrics.size()); - assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); - } - - @Test - public void response_contains_service_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void all_consumers_get_health_from_service_that_is_down() { - assertDownServiceHealth(DEFAULT_CONSUMER); - assertDownServiceHealth(CUSTOM_CONSUMER); - } - - @Test - public void all_timestamps_are_equal_and_non_zero() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - Long nodeTimestamp = jsonModel.node.timestamp; - assertNotEquals(0L, (long) nodeTimestamp); - for (var service : jsonModel.services) - assertEquals(nodeTimestamp, service.timestamp); - } - - @Test - public void custom_consumer_gets_only_its_whitelisted_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER); - - assertNotNull(jsonModel.node); - // TODO: see comment in ExternalMetrics.setExtraMetrics - // assertEquals(0, jsonModel.node.metrics.size()); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals("custom-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals("custom-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void invalid_path_yields_error_response() throws Exception { - String response = testDriver.sendRequest(V1_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("error")); - } - - private void assertDownServiceHealth(String consumer) { - GenericJsonModel jsonModel = getResponseAsJsonModel(consumer); - - GenericService downService = jsonModel.services.get(1); - assertEquals(DOWN.status, downService.status.code); - assertEquals("No response", downService.status.description); - - // Service should output metric dimensions, even without metrics, because they contain important info about the service. - assertEquals(1, downService.metrics.size()); - assertEquals(0, downService.metrics.get(0).values.size()); - assertFalse(downService.metrics.get(0).dimensions.isEmpty()); - assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id)); - } - - private String replaceTimestamps(String s) { - return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + @Before + public void initModelClass() { + modelClass = GenericJsonModel.class; } - private static GenericMetrics getMetricsForInstance(String instance, GenericService service) { - for (var metrics : service.metrics) { - if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance)) - return metrics; - } - fail("Could not find metrics for service instance " + instance); - throw new RuntimeException(); + @Override + GenericJsonModel getGenericJsonModel(GenericJsonModel genericJsonModel) { + return genericJsonModel; } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java new file mode 100644 index 00000000000..27ee6be4be3 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.concurrent.Executors; + +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.V2_PATH; +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.VALUES_PATH; + +/** + * @author gjoranv + */ +@SuppressWarnings("UnstableApiUsage") +public class MetricsV2HandlerTest extends MetricsHandlerTestBase<GenericApplicationModel> { + + private static final String V2_URI = URI_BASE + V2_PATH; + private static final String VALUES_URI = URI_BASE + VALUES_PATH; + + + @BeforeClass + public static void setup() { + rootUri = V2_URI; + valuesUri = VALUES_URI; + var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), + getMetricsManager(), + vespaServices, + getMetricsConsumers(), + nodeInfoConfig()); + testDriver = new RequestHandlerTestDriver(handler); + } + + @Before + public void initModelClass() { + modelClass = GenericApplicationModel.class; + } + + @Override + GenericJsonModel getGenericJsonModel(GenericApplicationModel genericApplicationModel) { + return genericApplicationModel.nodes.get(0); + } + + private static NodeInfoConfig nodeInfoConfig() { + return new NodeInfoConfig.Builder() + .role("my-role") + .hostname("my-hostname") + .build(); + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index a85f0425b4b..a224c4090b3 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.util.concurrent.Executors; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -83,7 +84,7 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase { @Test public void service_metrics_have_configured_dimensions() { String dummy0 = getLine(valuesResponse, DummyService.NAME + "0"); - assertTrue(dummy0.contains("consumer_dim=\"default-val\"")); + assertTrue(dummy0.contains(REASON + "=\"default-val\"")); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java index 9b37a805245..78c80689299 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java @@ -14,6 +14,7 @@ import java.util.Map; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -45,13 +46,23 @@ public class MetricsPacketTest { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .statusCode(0) .statusMessage("") - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) .build(); assertEquals(1, packet.consumers().size()); } @Test + public void builder_allows_inspecting_consumers() { + var consumer = toConsumerId("my-consumer"); + var builder = new MetricsPacket.Builder(toServiceId("foo")) + .statusCode(0) + .statusMessage("") + .addConsumers(singleton(consumer)); + assertTrue(builder.hasConsumer(consumer)); + } + + @Test public void builder_can_retain_subset_of_metrics() { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .putMetrics(ImmutableList.of( diff --git a/metrics-proxy/src/test/resources/generic-sample.json b/metrics-proxy/src/test/resources/generic-sample.json index de617895f86..c9b02696e69 100644 --- a/metrics-proxy/src/test/resources/generic-sample.json +++ b/metrics-proxy/src/test/resources/generic-sample.json @@ -33,7 +33,14 @@ "queries.count": 4 }, "dimensions": { - "documentType": "music" + "applicationId": "music.default", + "clustertype": "container", + "clusterid": "default", + "instance": "qrserver0", + "zone": "us-west", + "api": "search/", + "documenttype": "music", + "non-public": "will-be-removed" } } ] diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index 5a75c8b31ea..d465464de7f 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -8,6 +8,7 @@ "methods": [ "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)", "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, double)", + "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, java.lang.String)", "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(com.yahoo.tensor.Tensor)", "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(double)", "public com.yahoo.tensor.Tensor evaluate()", diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java index 994f6dd9b64..e373a54bcd1 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -2,6 +2,7 @@ package ai.vespa.models.evaluation; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -61,6 +62,21 @@ public class FunctionEvaluator { } /** + * Binds the given variable referred in this expression to the given value. + * String values are not yet supported in tensors. + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public FunctionEvaluator bind(String name, String value) { + if (evaluated) + throw new IllegalStateException("Cannot bind a new value in a used evaluator"); + context.put(name, new StringValue(value)); + return this; + } + + /** * Sets the default value to use for variables which are not bound * * @param value the default value diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index 5c353fcdf35..de23a8c6526 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -77,8 +77,14 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { property(request, missingValueKey).ifPresent(missingValue -> evaluator.setMissingValue(Tensor.from(missingValue))); for (Map.Entry<String, TensorType> argument : evaluator.function().argumentTypes().entrySet()) { - property(request, argument.getKey()).ifPresent(value -> evaluator.bind(argument.getKey(), - Tensor.from(argument.getValue(), value))); + Optional<String> value = property(request, argument.getKey()); + if (value.isPresent()) { + try { + evaluator.bind(argument.getKey(), Tensor.from(argument.getValue(), value.get())); + } catch (IllegalArgumentException e) { + evaluator.bind(argument.getKey(), value.get()); // since we don't yet support tensors with string values + } + } } Tensor result = evaluator.evaluate(); return new Response(200, JsonFormat.encode(result)); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java index 9320ac3fad8..0d13b7d4660 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java @@ -25,7 +25,7 @@ public class MlModelsImportingTest { public void testImportingModels() { ModelTester tester = new ModelTester("src/test/resources/config/models/"); - assertEquals(4, tester.models().size()); + assertEquals(5, tester.models().size()); // TODO: When we get type information in Models, replace the evaluator.context().names() check below by that { @@ -47,7 +47,24 @@ public class MlModelsImportingTest { } { + Model lightgbm = tester.models().get("lightgbm_regression"); + // Function + assertEquals(1, lightgbm.functions().size()); + ExpressionFunction function = tester.assertFunction("lightgbm_regression", + "(optimized sum of condition trees of size 480 bytes)", + lightgbm); + assertEquals("tensor()", function.returnType().get().toString()); + assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", commaSeparated(function.arguments())); + function.arguments().forEach(arg -> assertEquals(TensorType.empty, function.argumentTypes().get(arg))); + + // Evaluator + FunctionEvaluator evaluator = lightgbm.evaluatorOf(); + assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", evaluator.context().names().stream().sorted().collect(Collectors.joining(", "))); + assertEquals(1.91300868202, evaluator.evaluate().sum().asDouble(), delta); + } + + { Model onnxMnistSoftmax = tester.models().get("mnist_softmax"); // Function diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 629c82f410a..c9e49d3be02 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -56,7 +56,7 @@ public class ModelsEvaluationHandlerTest { public void testListModels() { String url = "http://localhost/model-evaluation/v1"; String expected = - "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\"}"; + "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; assertResponse(url, 200, expected); } @@ -94,6 +94,39 @@ public class ModelsEvaluationHandlerTest { } @Test + public void testLightGBMEvaluationWithoutBindings() { + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testLightGBMEvaluationWithBindings() { + Map<String, String> properties = new HashMap<>(); + properties.put("numerical_1", "0.1"); + properties.put("numerical_2", "0.2"); + properties.put("categorical_1", "a"); + properties.put("categorical_2", "i"); + properties.put("non-existing-binding", "-1"); + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":2.054697758469921}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test + public void testLightGBMEvaluationWithMissingValue() { + Map<String, String> properties = new HashMap<>(); + properties.put("missing-value", "-1.0"); + properties.put("numerical_2", "0.5"); + properties.put("categorical_1", "b"); + properties.put("categorical_2", "j"); + properties.put("non-existing-binding", "-1"); + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":2.0745534018208094}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test public void testMnistSoftmaxDetails() { String url = "http://localhost:8080/model-evaluation/v1/mnist_softmax"; String expected = "{\"model\":\"mnist_softmax\",\"functions\":[{\"function\":\"default.add\",\"info\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add\",\"eval\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add/eval\",\"arguments\":[{\"name\":\"Placeholder\",\"type\":\"tensor(d0[],d1[784])\"}]}]}"; diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg index c25c5ba555b..385115b7cd4 100644 --- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg +++ b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg @@ -26,3 +26,6 @@ rankprofile[3].fef.property[3].name "rankingExpression(serving_default.y).input. rankprofile[3].fef.property[3].value "tensor(d0[],d1[784])" rankprofile[3].fef.property[4].name "rankingExpression(serving_default.y).type" rankprofile[3].fef.property[4].value "tensor(d1[10])" +rankprofile[4].name "lightgbm_regression" +rankprofile[4].fef.property[0].name "rankingExpression(lightgbm_regression).rankingScript" +rankprofile[4].fef.property[0].value "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in ["k", "l", "m"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in ["d", "e"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in ["a", "b"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in ["k", "l", "m"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)" diff --git a/model-integration/src/main/config/model-integration.xml b/model-integration/src/main/config/model-integration.xml index 90ec7d7275e..34f5f0ce31a 100644 --- a/model-integration/src/main/config/model-integration.xml +++ b/model-integration/src/main/config/model-integration.xml @@ -1,11 +1,12 @@ <!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!-- Component which can import some ml model. This is included into the config server services.xml to enable it to translate - model pseudofeatures in ranking expressions during config mddel building. + model pseudo features in ranking expressions during config model building. It is provided as separate bundles instead of being included in the config model because some of these (TensorFlow) includes JNI code, and so can only exist in one instance in the server. --> <component id="ai.vespa.rankingexpression.importer.onnx.OnnxImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter" bundle="model-integration" /> +<component id="ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.vespa.VespaImporter" bundle="model-integration" /> diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java index d22a8067bd4..c7f320ed3b4 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java @@ -224,6 +224,11 @@ public class DimensionRenamer { /** Returns whether this is an opposite of another constraint */ boolean isOpposite() { return opposite; } + public static Constraint equal() { return new EqualConstraint(false, false); } + public static Constraint notEqual() { return new NotEqualConstraint(false, false); } + public static Constraint lessThan() { return new LessThanConstraint(false, false); } + public static Constraint greaterThan() { return new GreaterThanConstraint(false, false); } + public static Constraint equal(boolean soft) { return new EqualConstraint(soft, false); } public static Constraint notEqual(boolean soft) { return new NotEqualConstraint(soft, false); } public static Constraint lessThan(boolean soft) { return new LessThanConstraint(soft, false); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java new file mode 100644 index 00000000000..76caa652ad2 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java @@ -0,0 +1,54 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import ai.vespa.rankingexpression.importer.ImportedModel; +import ai.vespa.rankingexpression.importer.ModelImporter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; + +import java.io.File; +import java.io.IOException; + +/** + * Converts a LightGBM model into a ranking expression. + * + * @author lesters + */ +public class LightGBMImporter extends ModelImporter { + + @Override + public boolean canImport(String modelPath) { + File modelFile = new File(modelPath); + if ( ! modelFile.isFile()) return false; + return modelFile.toString().endsWith(".json") && probe(modelFile); + } + + /** + * Returns true if the give file looks like a LightGBM json file. + * Currently, we just check if the json has an element called "tree_info" + */ + private boolean probe(File modelFile) { + try { + return new ObjectMapper().readTree(modelFile).has("tree_info"); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read '" + modelFile + "'", e); + } + } + + @Override + public ImportedModel importModel(String modelName, String modelPath) { + try { + ImportedModel model = new ImportedModel(modelName, modelPath); + LightGBMParser parser = new LightGBMParser(modelPath); + RankingExpression expression = new RankingExpression(parser.toRankingExpression()); + model.expression(modelName, expression); + return model; + } catch (IOException e) { + throw new IllegalArgumentException("Could not import LightGBM model from '" + modelPath + "'", e); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse ranking expression resulting from '" + modelPath + "'", e); + } + } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java new file mode 100644 index 00000000000..dc76ed8cb6f --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java @@ -0,0 +1,67 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +/** + * @author lesters + */ +public class LightGBMNode { + + // split nodes + private int split_feature; + private String threshold; // double for numerical, string for categorical + private String decision_type; + private boolean default_left; + private String missing_type; + private int internal_count; + private LightGBMNode left_child; + private LightGBMNode right_child; + + // leaf nodes + private double leaf_value; + private int leaf_count; + + public int getSplit_feature() { + return split_feature; + } + + public String getThreshold() { + return threshold; + } + + public String getDecision_type() { + return decision_type; + } + + public boolean isDefault_left() { + return default_left; + } + + public String getMissing_type() { + return missing_type; + } + + public int getInternal_count() { + return internal_count; + } + + public LightGBMNode getLeft_child() { + return left_child; + } + + public LightGBMNode getRight_child() { + return right_child; + } + + public double getLeaf_value() { + return leaf_value; + } + + public int getLeaf_count() { + return leaf_count; + } + + public boolean isLeaf() { + return left_child == null && right_child == null; + } + +}
\ No newline at end of file diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java new file mode 100644 index 00000000000..996343674ce --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java @@ -0,0 +1,146 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** + * @author lesters + */ +class LightGBMParser { + + private final String objective; + private final List<LightGBMNode> nodes; + private final List<String> featureNames; + private final Map<Integer, List<String>> categoryValues; // pr feature index + + LightGBMParser(String filePath) throws JsonProcessingException, IOException { + ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + JsonNode root = mapper.readTree(new File(filePath)); + + objective = root.get("objective").asText("regression"); + featureNames = parseFeatureNames(root); + nodes = parseTrees(mapper, root); + categoryValues = parseCategoryValues(root); + } + + private List<String> parseFeatureNames(JsonNode root) { + List<String> features = new ArrayList<>(); + for (JsonNode name : root.get("feature_names")) { + features.add(name.textValue()); + } + return features; + } + + private List<LightGBMNode> parseTrees(ObjectMapper mapper, JsonNode root) throws JsonProcessingException { + List<LightGBMNode> nodes = new ArrayList<>(); + for (JsonNode treeNode : root.get("tree_info")) { + nodes.add(mapper.treeToValue(treeNode.get("tree_structure"), LightGBMNode.class)); + } + return nodes; + } + + private Map<Integer, List<String>> parseCategoryValues(JsonNode root) { + Map<Integer, List<String>> categoryValues = new HashMap<>(); + + // Since the JSON format does not explicitly tell which features are + // categorical, we traverse the decision tree looking for categorical + // decisions and use that to determine which categorical features. + Set<Integer> categoricalFeatures = new TreeSet<>(); + nodes.forEach(node -> findCategoricalFeatures(node, categoricalFeatures)); + + // Again, the LightGBM JSON format does not explicitly tell which + // categorical values map to each categorical feature. The assumption + // here is that the order they appear in the "pandas_categorical" + // structure is the same order as the "feature_names". + var pandasFeatureIterator = root.get("pandas_categorical").iterator(); + var categoricalFeatureIterator = categoricalFeatures.iterator(); + while (pandasFeatureIterator.hasNext() && categoricalFeatureIterator.hasNext()) { + List<String> values = new ArrayList<>(); + pandasFeatureIterator.next().forEach(value -> values.add(value.textValue())); + categoryValues.put(categoricalFeatureIterator.next(), values); + } + + return categoryValues; + } + + private void findCategoricalFeatures(LightGBMNode node, Set<Integer> categoricalFeatures) { + if (node == null || node.isLeaf()) { + return; + } + if (node.getDecision_type().equals("==")) { + categoricalFeatures.add(node.getSplit_feature()); + } + findCategoricalFeatures(node.getLeft_child(), categoricalFeatures); + findCategoricalFeatures(node.getRight_child(), categoricalFeatures); + } + + String toRankingExpression() { + return applyObjective(nodes.stream().map(this::nodeToRankingExpression).collect(Collectors.joining(" + \n"))); + } + + // See https://lightgbm.readthedocs.io/en/latest/Parameters.html#objective + private String applyObjective(String expression) { + if (objective.startsWith("binary") || objective.equals("cross_entropy")) { + return "sigmoid(" + expression + ")"; + } + if (objective.equals("poisson") || objective.equals("gamma") || objective.equals("tweedie")) { + return "exp(" + expression + ")"; + } + return expression; // else: use expression directly + } + + private String nodeToRankingExpression(LightGBMNode node) { + if (node.isLeaf()) { + return Double.toString(node.getLeaf_value()); + } else { + String condition; + String feature = featureNames.get(node.getSplit_feature()); + if (node.getDecision_type().equals("==")) { + String values = transformCategoryIndexesToValues(node); + if (node.isDefault_left()) { // means go left (true) when isNan + condition = "isNan(" + feature + ") || (" + feature + " in [ " + values + "])"; + } else { + condition = feature + " in [" + values + "]"; + } + } else { // assumption: all other decision types are <= + double value = Double.parseDouble(node.getThreshold()); + if (node.isDefault_left()) { + condition = "!(" + feature + " >= " + value + ")"; + } else { + condition = feature + " < " + value; + } + } + String left = nodeToRankingExpression(node.getLeft_child()); + String right = nodeToRankingExpression(node.getRight_child()); + return "if (" + condition + ", " + left + ", " + right + ")"; + } + } + + private String transformCategoryIndexesToValues(LightGBMNode node) { + return Arrays.stream(node.getThreshold().split("\\|\\|")) + .map(index -> "\"" + transformCategoryIndexToValue(node.getSplit_feature(), index) + "\"") + .collect(Collectors.joining(",")); + } + + private String transformCategoryIndexToValue(int featureIndex, String valueIndex) { + if ( ! categoryValues.containsKey(featureIndex) ) { + return valueIndex; // We don't have a pandas categorical lookup table + } + return categoryValues.get(featureIndex).get(Integer.parseInt(valueIndex)); + } + +}
\ No newline at end of file diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java new file mode 100644 index 00000000000..b29145ee21b --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java index 8caa158e5be..b272d4c6750 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java @@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.OrderedTensorType; import ai.vespa.rankingexpression.importer.operations.IntermediateOperation; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import onnx.Onnx; @@ -37,6 +38,7 @@ class AttributeConverter implements IntermediateOperation.AttributeMap { case INT: return Optional.of(DoubleValue.frozen(attr.getI())); case FLOAT: return Optional.of(DoubleValue.frozen(attr.getF())); case STRING: return Optional.of(StringValue.frozen(attr.getS().toString())); + case TENSOR: return Optional.of(new TensorValue(TensorConverter.toVespaTensor(attr.getT(), TypeConverter.typeFrom(attr.getT())))); default: return Optional.empty(); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java index d42338deaf8..ffc64c38f16 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java @@ -2,13 +2,18 @@ package ai.vespa.rankingexpression.importer.onnx; +import ai.vespa.rankingexpression.importer.operations.ExpandDims; +import ai.vespa.rankingexpression.importer.operations.Gather; +import ai.vespa.rankingexpression.importer.operations.OnnxCast; import ai.vespa.rankingexpression.importer.operations.Gemm; import ai.vespa.rankingexpression.importer.operations.ConcatReduce; import ai.vespa.rankingexpression.importer.operations.OnnxConcat; import ai.vespa.rankingexpression.importer.operations.Reduce; import ai.vespa.rankingexpression.importer.operations.Select; +import ai.vespa.rankingexpression.importer.operations.Slice; import ai.vespa.rankingexpression.importer.operations.Softmax; import ai.vespa.rankingexpression.importer.operations.Squeeze; +import ai.vespa.rankingexpression.importer.operations.Unsqueeze; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import ai.vespa.rankingexpression.importer.IntermediateGraph; @@ -67,6 +72,7 @@ class GraphImporter { case "add": return new Join(modelName, nodeName, inputs, ScalarFunctions.add()); case "asin": return new Map(modelName, nodeName, inputs, ScalarFunctions.asin()); case "atan": return new Map(modelName, nodeName, inputs, ScalarFunctions.atan()); + case "cast": return new OnnxCast(modelName, nodeName, inputs, attributes); case "ceil": return new Map(modelName, nodeName, inputs, ScalarFunctions.ceil()); case "concat": return new OnnxConcat(modelName, nodeName, inputs, attributes); case "cos": return new Map(modelName, nodeName, inputs, ScalarFunctions.cos()); @@ -75,6 +81,7 @@ class GraphImporter { case "equal": return new Join(modelName, nodeName, inputs, ScalarFunctions.equal()); case "exp": return new Map(modelName, nodeName, inputs, ScalarFunctions.exp()); case "floor": return new Map(modelName, nodeName, inputs, ScalarFunctions.floor()); + case "gather": return new Gather(modelName, nodeName, inputs, attributes); case "gemm": return new Gemm(modelName, nodeName, inputs, attributes); case "greater": return new Join(modelName, nodeName, inputs, ScalarFunctions.greater()); case "identity": return new Identity(modelName, nodeName, inputs); @@ -105,6 +112,7 @@ class GraphImporter { case "shape": return new Shape(modelName, nodeName, inputs); case "sigmoid": return new Map(modelName, nodeName, inputs, ScalarFunctions.sigmoid()); case "sin": return new Map(modelName, nodeName, inputs, ScalarFunctions.sin()); + case "slice": return new Slice(modelName, nodeName, inputs, attributes); case "softmax": return new Softmax(modelName, nodeName, inputs, attributes); case "sub": return new Join(modelName, nodeName, inputs, ScalarFunctions.subtract()); case "squeeze": return new Squeeze(modelName, nodeName, inputs, attributes); @@ -113,6 +121,7 @@ class GraphImporter { case "where": return new Select(modelName, nodeName, inputs); case "tan": return new Map(modelName, nodeName, inputs, ScalarFunctions.tan()); case "tanh": return new Map(modelName, nodeName, inputs, ScalarFunctions.tanh()); + case "unsqueeze": return new Unsqueeze(modelName, nodeName, inputs, attributes); } IntermediateOperation op = new NoOp(modelName, nodeName, inputs); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java index 69d18d0ffcb..f8c7dc15857 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java @@ -37,12 +37,14 @@ class TensorConverter { case BOOL: return new RawBoolValues(tensorProto); case FLOAT: return new RawFloatValues(tensorProto); case DOUBLE: return new RawDoubleValues(tensorProto); + case INT32: return new RawIntValues(tensorProto); case INT64: return new RawLongValues(tensorProto); } } else { switch (tensorProto.getDataType()) { case FLOAT: return new FloatValues(tensorProto); case DOUBLE: return new DoubleValues(tensorProto); + case INT32: return new IntValues(tensorProto); case INT64: return new LongValues(tensorProto); } } @@ -96,6 +98,17 @@ class TensorConverter { @Override int size() { return size; } } + private static class RawIntValues extends RawValues { + private final IntBuffer values; + private final int size; + RawIntValues(Onnx.TensorProto tensorProto) { + values = bytes(tensorProto).asIntBuffer(); + size = values.remaining(); + } + @Override double get(int i) { return values.get(i); } + @Override int size() { return size; } + } + private static class RawLongValues extends RawValues { private final LongBuffer values; private final int size; @@ -125,6 +138,15 @@ class TensorConverter { @Override int size() { return tensorProto.getDoubleDataCount(); } } + private static class IntValues extends Values { + private final Onnx.TensorProto tensorProto; + IntValues(Onnx.TensorProto tensorProto) { + this.tensorProto = tensorProto; + } + @Override double get(int i) { return tensorProto.getInt32Data(i); } + @Override int size() { return tensorProto.getInt32DataCount(); } + } + private static class LongValues extends Values { private final Onnx.TensorProto tensorProto; LongValues(Onnx.TensorProto tensorProto) { diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java index 3487d889338..e02f29a63f9 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java @@ -40,7 +40,7 @@ public class ExpandDims extends IntermediateOperation { OrderedTensorType inputType = inputs.get(0).type().get(); int dimensionToInsert = (int)axis.asDouble(); if (dimensionToInsert < 0) { - dimensionToInsert = inputType.dimensions().size() - dimensionToInsert; + dimensionToInsert = inputType.dimensions().size() + dimensionToInsert; } OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType()); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java new file mode 100644 index 00000000000..2a34ae53d5e --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java @@ -0,0 +1,170 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.operations; + +import ai.vespa.rankingexpression.importer.DimensionRenamer; +import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.Slice; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar; + +/* + * Onnx gather is the same as Numpy take. + */ +public class Gather extends IntermediateOperation { + + private final AttributeMap attributeMap; + + private int axis; + + public Gather(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) { + super(modelName, nodeName, inputs); + this.attributeMap = attributeMap; + } + + @Override + protected OrderedTensorType lazyGetType() { + if ( ! allInputTypesPresent(2)) return null; + + OrderedTensorType dataType = inputs.get(0).type().get(); + OrderedTensorType indicesType = inputs.get(1).type().get(); + + axis = (int) attributeMap.get("axis").orElse(DoubleValue.zero).asDouble(); + if (axis < 0) + axis = dataType.rank() + axis; + + OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType()); + for (int i = 0; i < axis; ++i) { + addDimension(i, dataType.dimensions().get(i).size().orElse(-1L), typeBuilder); + } + for (int i = 0; i < indicesType.rank(); ++i) { + addDimension(i + axis, indicesType.dimensions().get(i).size().orElse(-1L), typeBuilder); + } + for (int i = axis + 1; i < dataType.rank(); ++i) { + addDimension(i + indicesType.rank(), dataType.dimensions().get(i).size().orElse(-1L), typeBuilder); + } + + inputs.get(0).exportAsRankingFunction = true; + inputs.get(1).exportAsRankingFunction = true; + + return typeBuilder.build(); + } + + private void addDimension(int dimensionIndex, long size, OrderedTensorType.Builder typeBuilder) { + String name = String.format("%s_%d", vespaName(), dimensionIndex); + typeBuilder.add(TensorType.Dimension.indexed(name, size)); + } + + @Override + protected TensorFunction lazyGetFunction() { + if ( ! allInputFunctionsPresent(2)) return null; + + IntermediateOperation data = inputs.get(0); + IntermediateOperation indices = inputs.get(1); + OrderedTensorType dataType = data.type().get(); + OrderedTensorType indicesType = indices.type().get(); + + String dataFunctionName = data.rankingExpressionFunctionName(); + String indicesFunctionName = indices.rankingExpressionFunctionName(); + + List<Slice.DimensionValue<Reference>> dataSliceDimensions = new ArrayList<>(); + for (int i = 0; i < axis; ++i) { + addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i); + } + + List<Slice.DimensionValue<Reference>> indicesSliceDimensions = new ArrayList<>(); + for (int i = 0; i < indicesType.rank(); ++i) { + addSliceDimension(indicesSliceDimensions, indicesType.dimensions().get(i).name(), axis + i); + } + ExpressionNode sliceExpression = createSliceExpression(indicesSliceDimensions, indicesFunctionName); + ExpressionNode indexExpression = createIndexExpression(dataType, sliceExpression); + addSliceDimension(dataSliceDimensions, dataType.dimensions().get(axis).name(), indexExpression); + + for (int i = axis + 1; i < dataType.rank(); ++i) { + addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i + indicesType.rank() - 1); + } + + sliceExpression = createSliceExpression(dataSliceDimensions, dataFunctionName); + return Generate.bound(type.type(), wrapScalar(sliceExpression)); + } + + private ExpressionNode createSliceExpression(List<Slice.DimensionValue<Reference>> dimensionValues, String referenceName) { + TensorFunction<Reference> inputIndices = new TensorFunctionNode.ExpressionTensorFunction(new ReferenceNode(referenceName)); + Slice<Reference> sliceIndices = new Slice<>(inputIndices, dimensionValues); + return new TensorFunctionNode(sliceIndices); + } + + /** to support negative indexing */ + private ExpressionNode createIndexExpression(OrderedTensorType dataType, ExpressionNode slice) { + ExpressionNode axisSize = new ConstantNode(new DoubleValue(dataType.dimensions().get(axis).size().get())); + ExpressionNode plus = new EmbracedNode(new ArithmeticNode(slice, ArithmeticOperator.PLUS, axisSize)); + ExpressionNode mod = new ArithmeticNode(plus, ArithmeticOperator.MODULO, axisSize); + return mod; + } + + private void addSliceDimension(List<Slice.DimensionValue<Reference>> dimensionValues, String dimensionName, ExpressionNode expr) { + dimensionValues.add(new Slice.DimensionValue<>(Optional.of(dimensionName), wrapScalar(new EmbracedNode(expr)))); + } + + private void addSliceDimension(List<Slice.DimensionValue<Reference>> dimensionValues, String dimensionName, int dimensionIndex) { + String outputDimensionName = type.dimensions().get(dimensionIndex).name(); + addSliceDimension(dimensionValues, dimensionName, new ReferenceNode(outputDimensionName)); + } + + @Override + public void addDimensionNameConstraints(DimensionRenamer renamer) { + if ( ! allInputTypesPresent(2)) return; + + for (int i = 0; i < type.dimensions().size(); i++) { + renamer.addDimension(type.dimensions().get(i).name()); + for (int j = i + 1; j < type.dimensions().size(); j++) { + renamer.addConstraint(type.dimensions().get(i).name(), type.dimensions().get(j).name(), + DimensionRenamer.Constraint.lessThan(), this); + } + } + + OrderedTensorType dataType = inputs.get(0).type().get(); + OrderedTensorType indicesType = inputs.get(1).type().get(); + + for (int i = 0; i < axis; ++i) { + renamer.addConstraint(type.dimensions().get(i).name(), + dataType.dimensions().get(i).name(), + DimensionRenamer.Constraint.equal(), this); + } + for (int i = 0; i < indicesType.rank(); ++i) { + renamer.addConstraint(type.dimensions().get(i + axis).name(), + indicesType.dimensions().get(i).name(), + DimensionRenamer.Constraint.equal(), this); + } + for (int i = axis + 1; i < dataType.rank(); ++i) { + renamer.addConstraint(type.dimensions().get(i + indicesType.rank() - 1).name(), + dataType.dimensions().get(i).name(), + DimensionRenamer.Constraint.equal(), this); + } + + } + + @Override + public Gather withInputs(List<IntermediateOperation> inputs) { + return new Gather(modelName(), name(), inputs, attributeMap); + } + + @Override + public String operationName() { return "Gather"; } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java index 724b5c6b3ac..2aa8b2a0d48 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java @@ -45,6 +45,7 @@ public abstract class IntermediateOperation { protected OrderedTensorType type; protected TensorFunction function; protected TensorFunction rankingExpressionFunction = null; + protected boolean exportAsRankingFunction = false; private final List<String> importWarnings = new ArrayList<>(); private Value constantValue = null; @@ -78,7 +79,7 @@ public abstract class IntermediateOperation { if (isConstant()) { ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName())); function = new TensorFunctionNode.ExpressionTensorFunction(constant); - } else if (outputs.size() > 1) { + } else if (outputs.size() > 1 || exportAsRankingFunction) { rankingExpressionFunction = lazyGetFunction(); function = new VariableTensor(rankingExpressionFunctionName(), type.type()); } else { @@ -137,7 +138,7 @@ public abstract class IntermediateOperation { return Optional.of(constantValue); } if (constantValueFunction != null) { - return Optional.of(constantValueFunction.apply(type)); + return Optional.of(constantValueFunction.apply(type().orElse(null))); } return Optional.empty(); } @@ -188,7 +189,7 @@ public abstract class IntermediateOperation { throw new IllegalArgumentException("Attempted to evaluate non-constant operation as a constant."); } Value val = evaluateAsConstant(new MapContext(DoubleValue.NaN)); - if ( ! val.asTensor().type().equals(type.type()) ) { + if (type != null && ! val.asTensor().type().equals(type.type()) ) { throw new IllegalArgumentException("Constant evaluation in " + name + " resulted in wrong type. " + "Expected: " + type.type() + " Got: " + val.asTensor().type()); } @@ -211,6 +212,9 @@ public abstract class IntermediateOperation { result = new TensorValue(lazyGetFunction().evaluate(context)); } context.put(constantName, result); + if (outputs.size() > 1 || exportAsRankingFunction) { + context.put(rankingExpressionFunctionName(), result); + } } return result; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java new file mode 100644 index 00000000000..d15ac1b69f7 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java @@ -0,0 +1,82 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.operations; + +import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.tensor.functions.TensorFunction; +import onnx.Onnx.TensorProto.DataType; + +import java.util.List; +import java.util.function.DoubleUnaryOperator; + +public class OnnxCast extends IntermediateOperation { + + private final AttributeMap attributeMap; + private final DataType toType; + + public OnnxCast(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) { + super(modelName, nodeName, inputs); + this.attributeMap = attributeMap; + if (attributeMap.get("to").isEmpty()) { + throw new IllegalArgumentException("OnnxCast in " + name + ": Required attribute 'to' is missing."); + } + toType = DataType.forNumber((int) attributeMap.get("to").get().asDouble()); + } + + @Override + protected OrderedTensorType lazyGetType() { + if (!allInputTypesPresent(1)) + return null; + return inputs.get(0).type().orElse(null); + } + + @Override + protected TensorFunction lazyGetFunction() { + if ( ! allInputFunctionsPresent(1)) + return null; + TensorFunction input = inputs.get(0).function().get(); + switch (toType) { + case BOOL: + return new com.yahoo.tensor.functions.Map(input, new AsBool()); + case INT8: + case INT16: + case INT32: + case INT64: + case UINT8: + case UINT16: + case UINT32: + case UINT64: + return new com.yahoo.tensor.functions.Map(input, new AsInt()); + case FLOAT: + case DOUBLE: + case FLOAT16: + return input; + case STRING: + throw new IllegalArgumentException("OnnxCast in " + name + ": Casting to string is not implemented."); + default: + throw new IllegalArgumentException("OnnxCast in " + name + ": Unknown or undefined cast: " + toType.name()); + } + } + + @Override + public OnnxCast withInputs(List<IntermediateOperation> inputs) { + return new OnnxCast(modelName(), name(), inputs, attributeMap); + } + + @Override + public String operationName() { return "Cast"; } + + private static class AsBool implements DoubleUnaryOperator { + @Override + public double applyAsDouble(double operand) { return operand != 0.0 ? 1 : 0; } + @Override + public String toString() { return "f(a)(a!=0)"; } + } + + private static class AsInt implements DoubleUnaryOperator { + @Override + public double applyAsDouble(double operand) { return operand < 0 ? Math.ceil(operand) : Math.floor(operand); } + @Override + public String toString() { return "f(a)(if (a < 0, ceil(a), floor(a)))"; } + } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java new file mode 100644 index 00000000000..e5463291ef8 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java @@ -0,0 +1,203 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.operations; + +import ai.vespa.rankingexpression.importer.DimensionRenamer; +import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; +import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar; + +/** + * Onnx slice operation. + * + * Opset 1 to 9 accepts starts, ends, and axes tensors as attributes + * + * Opset 10 and up accepts starts, ends, axes, and steps tensors as inputs. Here we assume these are + * constants, otherwise we can't import this model because that would mean we + * would not know the resulting tensor type until run-time, and that is currently + * not supported in Vespa. + */ +public class Slice extends IntermediateOperation { + + private final AttributeMap attributes; + + private int[] starts; + private int[] ends; + private int[] steps; + + public Slice(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributes) { + super(modelName, nodeName, inputs); + this.attributes = attributes; + } + + @Override + protected OrderedTensorType lazyGetType() { + if (inputs.size() < 1 || inputs.get(0).type().isEmpty()) { + return null; + } + OrderedTensorType dataType = inputs.get(0).type().get(); + + // required as we use tensor create + inputs.get(0).exportAsRankingFunction = true; + + // Todo: only supports opsets 1-9, for >= get these from inputs + int[] startsInput = attributeListAsArray("starts", 0); + int[] endsInput = attributeListAsArray("ends", 0); + int[] stepsInput = new int[dataType.rank()]; Arrays.fill(stepsInput, 1); // Todo: get from input when opset >= 10 + + int[] axes; + if (attributes.getList("axes").isPresent()) { + axes = attributeListAsArray("axes", 0); + } else { + // infer axes: default is [0, 1, ..., len('starts')-1] + axes = new int[startsInput.length]; + for (int i = 0; i < startsInput.length; ++i) { + axes[i] = i; + } + } + + if (startsInput.length != endsInput.length) { + throw new IllegalArgumentException("Slice in " + name + ": 'starts' and 'ends' indexes are not of the same size."); + } + if (startsInput.length != axes.length) { + throw new IllegalArgumentException("Slice in " + name + ": 'axes' and 'starts' are not of same size."); + } + + int[] dimensionSizes = new int [dataType.rank()]; + for (int i = 0; i < dataType.rank(); ++i) { + dimensionSizes[i] = dataType.dimensions().get(i).size().get().intValue(); + } + + starts = new int[dataType.rank()]; Arrays.fill(starts, 0); + ends = new int[dataType.rank()]; + steps = new int[dataType.rank()]; Arrays.fill(steps, 1); + + for (int i = 0; i < axes.length; ++i) { + int axis = axes[i]; + int start = startsInput[i]; + int end = endsInput[i]; + int step = stepsInput[i]; + + axis = Math.min(axis, dataType.rank() - 1); + axis = axis < 0 ? axis + dataType.rank() : axis; + + start = Math.min(start, dimensionSizes[axis]); + start = start < 0 ? start + dimensionSizes[axis] : start; + + end = Math.min(end, dimensionSizes[axis]); + end = end < 0 ? end + dimensionSizes[axis] : end; + + // Todo: check negative values for step size + + starts[axis] = start; + steps[axis] = step; + + if (step == 0) { + throw new IllegalArgumentException("Slice in " + name + ": illegal step size of 0."); + } + if ((end - start) < 1) { + throw new IllegalArgumentException("Slice in " + name + ": illegal start (" + start + ") and end (" + end + ") index."); + } + dimensionSizes[axis] = (end - start) / step; + } + + OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType()); + for (int i = 0; i < dataType.rank(); ++i) { + addDimension(i, dimensionSizes[i], typeBuilder); + } + return typeBuilder.build(); + } + + private int[] attributeListAsArray(String name, int defaultValue) { + if (attributes.getList(name).isEmpty()) { + throw new IllegalArgumentException("Slice in " + name + ": Required attribute '" + name + "' is missing."); + } + List<Value> list = attributes.getList(name).get(); + int[] result = new int[list.size()]; Arrays.fill(result, defaultValue); + for (int i = 0; i < list.size(); ++i) { + result[i] = (int)list.get(i).asDouble(); + } + return result; + } + + private void addDimension(int dimensionIndex, long size, OrderedTensorType.Builder typeBuilder) { + String name = String.format("%s_%d", vespaName(), dimensionIndex); + typeBuilder.add(TensorType.Dimension.indexed(name, size)); + } + + @Override + protected TensorFunction lazyGetFunction() { + if (inputs.size() < 1 || inputs.get(0).function().isEmpty()) { + return null; + } + + IntermediateOperation data = inputs.get(0); + OrderedTensorType dataType = data.type().get(); + String dataFunctionName = data.rankingExpressionFunctionName(); + + List<com.yahoo.tensor.functions.Slice.DimensionValue<Reference>> dimensionValues = new ArrayList<>(); + + for (int axis = 0; axis < dataType.rank(); ++axis) { + int start = starts[axis]; + int step = steps[axis]; + + String inputDimensionName = dataType.dimensions().get(axis).name(); + String outputDimensionName = type.dimensions().get(axis).name(); + + ExpressionNode stepSize = new ConstantNode(new DoubleValue(step)); + ExpressionNode startIndex = new ConstantNode(new DoubleValue(start)); + + // step * (d0 + start) + ExpressionNode reference = new ReferenceNode(outputDimensionName); + ExpressionNode plus = new EmbracedNode(new ArithmeticNode(reference, ArithmeticOperator.PLUS, startIndex)); + ExpressionNode mul = new ArithmeticNode(stepSize, ArithmeticOperator.MULTIPLY, plus); + + dimensionValues.add(new com.yahoo.tensor.functions.Slice.DimensionValue<>(Optional.of(inputDimensionName), wrapScalar(new EmbracedNode(mul)))); + } + + TensorFunction<Reference> inputIndices = new TensorFunctionNode.ExpressionTensorFunction(new ReferenceNode(dataFunctionName)); + com.yahoo.tensor.functions.Slice<Reference> sliceIndices = new com.yahoo.tensor.functions.Slice<>(inputIndices, dimensionValues); + ExpressionNode sliceExpression = new TensorFunctionNode(sliceIndices); + + TensorFunction generate = Generate.bound(type.type(), wrapScalar(sliceExpression)); + return generate; + } + + @Override + public void addDimensionNameConstraints(DimensionRenamer renamer) { + // Todo: what to do? + for (int i = 0; i < type.dimensions().size(); i++) { + renamer.addDimension(type.dimensions().get(i).name()); + for (int j = i + 1; j < type.dimensions().size(); j++) { + renamer.addConstraint(type.dimensions().get(i).name(), type.dimensions().get(j).name(), + DimensionRenamer.Constraint.lessThan(), this); + } + } + } + + @Override + public Slice withInputs(List<IntermediateOperation> inputs) { + return new Slice(modelName(), name(), inputs, attributes); + } + + @Override + public String operationName() { return "Slice"; } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java new file mode 100644 index 00000000000..0df09c21530 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java @@ -0,0 +1,109 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.operations; + +import ai.vespa.rankingexpression.importer.DimensionRenamer; +import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.Generate; +import com.yahoo.tensor.functions.ScalarFunctions; +import com.yahoo.tensor.functions.TensorFunction; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class Unsqueeze extends IntermediateOperation { + + private final AttributeMap attributeMap; + private List<String> expandDimensions; + + public Unsqueeze(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) { + super(modelName, nodeName, inputs); + this.attributeMap = attributeMap; + if (attributeMap.getList("axes").isEmpty()) { + throw new IllegalArgumentException("Unsqueeze in " + name + ": Required attribute 'axes' is missing."); + } + } + + @Override + protected OrderedTensorType lazyGetType() { + if ( ! allInputTypesPresent(1)) return null; + + OrderedTensorType inputType = inputs.get(0).type().get(); + Set<Integer> dimensionsToInsert = attributeMap.getList("axes").get().stream(). + map(d -> (int)d.asDouble()).collect(Collectors.toSet()); + + // handle negative dimension indexes + int rank = inputType.rank() + dimensionsToInsert.size(); + dimensionsToInsert = dimensionsToInsert.stream().map(d -> d < 0 ? rank + d : d).collect(Collectors.toSet()); + + expandDimensions = new ArrayList<>(); + OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType()); + int inputDimensionIndex = 0; + for (int expandedDimensionIndex = 0; expandedDimensionIndex < rank; ++expandedDimensionIndex) { + if (dimensionsToInsert.contains(expandedDimensionIndex)) { + addDimension(expandedDimensionIndex, typeBuilder); + } else { + typeBuilder.add(inputType.dimensions().get(inputDimensionIndex)); + inputDimensionIndex++; + } + } + return typeBuilder.build(); + } + + private void addDimension(int dimensionIndex, OrderedTensorType.Builder typeBuilder) { + String name = String.format("%s_%d", vespaName(), dimensionIndex); + expandDimensions.add(name); + typeBuilder.add(TensorType.Dimension.indexed(name, 1L)); + } + + @Override + protected TensorFunction lazyGetFunction() { + if ( ! allInputFunctionsPresent(1)) return null; + + // multiply with a generated tensor created from the expanded dimensions + TensorType.Builder typeBuilder = new TensorType.Builder(resultValueType()); + for (String name : expandDimensions) { + typeBuilder.indexed(name, 1); + } + TensorType generatedType = typeBuilder.build(); + ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); + Generate generatedFunction = new Generate(generatedType, + new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); + return new com.yahoo.tensor.functions.Join(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply()); + } + + @Override + public void addDimensionNameConstraints(DimensionRenamer renamer) { + addConstraintsFrom(type, renamer); + } + + @Override + public void renameDimensions(DimensionRenamer renamer) { + super.renameDimensions(renamer); + List<String> renamedDimensions = new ArrayList<>(expandDimensions.size()); + for (String name : expandDimensions) { + Optional<String> newName = renamer.dimensionNameOf(name); + if (newName.isEmpty()) { + return; // presumably, already renamed + } + renamedDimensions.add(newName.get()); + } + expandDimensions = renamedDimensions; + } + + @Override + public Unsqueeze withInputs(List<IntermediateOperation> inputs) { + return new Unsqueeze(modelName(), name(), inputs, attributeMap); + } + + @Override + public String operationName() { return "Unsqueeze"; } + +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java new file mode 100644 index 00000000000..d2ef4b984ff --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer; +import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestNode; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * @author lesters + */ +public class LightGBMImportEvaluationTestCase extends LightGBMTestBase { + + @Test + public void testRegression() { + RankingExpression expression = importModel("src/test/models/lightgbm/regression.json"); + ArrayContext context = new ArrayContext(expression, true, DoubleValue.NaN); + + assertEvaluation(1.91300868, expression, features(context)); + assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + + ExpressionOptimizer optimizer = new ExpressionOptimizer(); + optimizer.optimize(expression, (ContextIndex)context); + assertTrue(expression.getRoot() instanceof GBDTForestNode); + + assertEvaluation(1.91300868, expression, features(context)); + assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + } + + @Test + public void testClassification() { + RankingExpression expression = importModel("src/test/models/lightgbm/classification.json"); + ArrayContext context = new ArrayContext(expression, DoubleValue.NaN); + assertEvaluation(0.37464997, expression, features(context)); + assertEvaluation(0.37464997, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(0.38730827, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(0.5647872, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + } + +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java new file mode 100644 index 00000000000..80c2ce68394 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; + +import static org.junit.Assert.assertEquals; + +/** + * @author lesters + */ +class LightGBMTestBase { + + RankingExpression importModel(String path) { + return new LightGBMImporter().importModel("lightgbm", path).expressions().get("lightgbm"); + } + + void assertEvaluation(double expected, RankingExpression expr, TestFeatures features) { + assertEquals(expected, expr.evaluate(features.context).asDouble(), 1e-6); + } + + TestFeatures features(ArrayContext context) { + return new TestFeatures(context.clone()); + } + + static class TestFeatures { + private final ArrayContext context; + TestFeatures(ArrayContext context) { + this.context = context; + } + TestFeatures add(String name, double value) { + context.put(name, value); + return this; + } + TestFeatures add(String name, String value) { + context.put(name, new StringValue(value)); + return this; + } + } + +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java index 6954abe5157..94c5577357b 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java @@ -17,6 +17,7 @@ import com.yahoo.tensor.functions.ConstantTensor; import com.yahoo.tensor.functions.Rename; import com.yahoo.tensor.functions.TensorFunction; import onnx.Onnx; +import org.junit.Ignore; import org.junit.Test; import java.util.ArrayList; @@ -26,7 +27,9 @@ import static ai.vespa.rankingexpression.importer.onnx.GraphImporter.*; import static onnx.Onnx.AttributeProto.AttributeType.FLOAT; import static onnx.Onnx.AttributeProto.AttributeType.INT; import static onnx.Onnx.AttributeProto.AttributeType.INTS; +import static onnx.Onnx.AttributeProto.AttributeType.TENSOR; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Unit tests for ONNX operators. The number on the test reflects the minimum @@ -294,6 +297,27 @@ public class OnnxOperationsTestCase { } @Test + public void testUnsqueeze1() throws ParseException { + Tensor x = evaluate("tensor(d0[2]):[1, 2]"); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {0})); + assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1]):[1, 2]"), createAttribute("axes", new int[] {1})); + assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1]):[1, 2]"), createAttribute("axes", new int[] {-1})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {-2})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {0,0})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1]):[1, 2]"), createAttribute("axes", new int[] {0,2})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1]):[1, 2]"), createAttribute("axes", new int[] {2,0})); + + x = evaluate("tensor(d0[2],d1[3]):[1,2,3,4,5,6]"); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[1],d2[2],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,1})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,2})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[3],d3[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,3})); + assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1],d2[1],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {1,2})); + assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[3],d2[1],d3[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {2,3})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3],d4[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,2,4})); + assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3],d4[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {4,2,0})); + } + + @Test public void testWhere9() throws ParseException { Tensor x = evaluate("tensor(d0[2],d1[2]):[1, 2, 3, 4]"); Tensor y = evaluate("tensor(d0[2],d1[2]):[5, 6, 7, 8]"); @@ -308,6 +332,109 @@ public class OnnxOperationsTestCase { assertEval("where", evaluate("tensor(d0[1],d1[1]):[1]"), x, y, x); } + @Test + public void testCast1() throws ParseException { + Tensor x = evaluate("tensor(d0[4]):[-1.9, 0.0, 1.1, 2.9]"); + assertEval("cast", x, evaluate("tensor(d0[4]):[1,0,1,1]"), createAttribute("to", 9)); // boolean + assertEval("cast", x, evaluate("tensor(d0[4]):[-1,0,1,2]"), createAttribute("to", 6)); // int32 + assertEval("cast", x, evaluate("tensor(d0[4]):[-1,0,1,2]"), createAttribute("to", 12)); // uint32 + assertEval("cast", x, evaluate("tensor(d0[4]):[-1.9,0,1.1,2.9]"), createAttribute("to", 1)); // float + try { + assertEval("cast", x, evaluate("tensor(d0[4]):[1,0,1,1]"), createAttribute("to", 8)); // string + fail(); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "OnnxCast in cast: Casting to string is not implemented."); + } + } + + @Test + public void testGather1() throws ParseException { + // 1 dim input, 1 dim indices + Tensor x = evaluate("tensor(d0[6]):[1,2,3,4,5,6]"); + Tensor y = evaluate("tensor(d0[3]):[0,2,4]"); + assertEval("gather", x, y, evaluate("tensor(d0[3]):[1,3,5]")); + + // 2 dim input, 1 dim indices - axis 0 + x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]"); + y = evaluate("tensor(d0[4]):[2, 1, 0, 2]"); + assertEval("gather", x, y, evaluate("tensor(d0[4],d1[2]):[5, 6, 3, 4, 1, 2, 5, 6]")); + + // 1 dim input, 2 dim indices - axis 0 + x = evaluate("tensor(d0[6]):[1, 2, 3, 4, 5, 6]"); + y = evaluate("tensor(d0[2],d1[2]):[0, 1, 3, 5]"); + assertEval("gather", x, y, evaluate("tensor(d0[2],d1[2]):[1, 2, 4, 6]")); + + // 2 dim input, 2 dim indices - axis 0 + x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]"); + y = evaluate("tensor(d0[2],d1[2]):[0, 1, 1, 2]"); + assertEval("gather", x, y, evaluate("tensor(d0[2],d1[2],d2[2]):[1, 2, 3, 4, 3, 4, 5, 6]"), createAttribute("axis", -2)); + + // 2 dim input, 1 dim indices - axis 1 + x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]"); + y = evaluate("tensor(d0[4]):[0, 1, 0, 1]"); + assertEval("gather", x, y, evaluate("tensor(d0[3],d1[4]):[1,2,1,2,3,4,3,4,5,6,5,6]"), createAttribute("axis", 1)); + + // 2 dim input, 2 dim indices - axis 1 + x = evaluate("tensor(d0[3],d1[3]):[1, 2, 3, 4, 5, 6, 7, 8, 9]"); + y = evaluate("tensor(d0[1],d1[2]):[0, 2]"); + assertEval("gather", x, y, evaluate("tensor(d0[3],d1[1],d2[2]):[1,3,4,6,7,9]"), createAttribute("axis", 1)); + + // 1 dim input, 1 dim indices - negative indices + x = evaluate("tensor(d0[6]):[1,2,3,4,5,6]"); + y = evaluate("tensor(d0[3]):[0,-2,-4]"); + assertEval("gather", x, y, evaluate("tensor(d0[3]):[1,5,3]")); + } + + @Test + public void testSlice1() throws ParseException { + Tensor x = evaluate("tensor(d0[2],d1[4]):[ [1,2,3,4],[5,6,7,8] ]"); + AttributeConverter attributes = createAttributes(). + attr("starts", new int[] {1, 0}). + attr("ends", new int[] {2, 3}). + attr("axes", new int[] {0, 1}).build(); + assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes); + + attributes = createAttributes(). + attr("starts", new int[] {0, 1}). + attr("ends", new int[] {-1, 1000}).build(); + assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [2,3,4] ]"), attributes); + + attributes = createAttributes(). + attr("starts", new int[] {0, 1}). + attr("ends", new int[] {3, 2}). + attr("axes", new int[] {1, 0}).build(); // axes are switched + assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes); + + attributes = createAttributes(). + attr("starts", new int[] {1, 0}). + attr("ends", new int[] {2, 3}). + attr("axes", new int[] {0, -1}).build(); // negative axes + assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes); + + attributes = createAttributes(). + attr("starts", new int[] {1}). + attr("ends", new int[] {2}). + attr("axes", new int[] {0}).build(); // axis 1 is not specified + assertEval("slice", x, evaluate("tensor(d0[1],d1[4]):[ [5,6,7,8] ]"), attributes); + + attributes = createAttributes(). + attr("starts", new int[] {0}). + attr("ends", new int[] {1}).build(); + assertEval("slice", x, evaluate("tensor(d0[1],d1[4]):[ [1,2,3,4] ]"), attributes); + } + + @Ignore + @Test + public void testSlice10() throws ParseException { + Tensor x = evaluate("tensor(d0[2],d1[4]):[ [1,2,3,4],[5,6,7,8] ]"); + Tensor starts = evaluate("tensor(d0[2]):[1,0]"); + Tensor ends = evaluate("tensor(d0[2]):[2,3]"); + Tensor axes = evaluate("tensor(d0[2]):[0,1]"); + Tensor steps = evaluate("tensor(d0[2]):[1,2]"); + assertEval("slice", x, starts, ends, axes, steps, evaluate("tensor(d0[1],d1[2]):[ [5,7] ]")); + + } + private Tensor evaluate(String expr) throws ParseException { return evaluate(expr, null, null, null); } @@ -334,28 +461,40 @@ public class OnnxOperationsTestCase { } private void assertEval(String opName, Tensor x, Tensor expected) { - assertEval(opName, x, null, null, expected, null); + assertEval(opName, x, null, null, null, null, expected, null); } private void assertEval(String opName, Tensor x, Tensor expected, AttributeConverter attr) { - assertEval(opName, x, null, null, expected, attr); + assertEval(opName, x, null, null, null, null, expected, attr); } private void assertEval(String opName, Tensor x, Tensor y, Tensor expected, AttributeConverter attr) { - assertEval(opName, x, y, null, expected, attr); + assertEval(opName, x, y, null, null, null, expected, attr); } private void assertEval(String opName, Tensor x, Tensor y, Tensor expected) { - assertEval(opName, x, y, null, expected, null); + assertEval(opName, x, y, null, null, null, expected, null); } private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor expected) { - assertEval(opName, x, y, z, expected, null); + assertEval(opName, x, y, z, null, null, expected, null); } private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor expected, AttributeConverter attr) { + assertEval(opName, x, y, z, null, null, expected, attr); + } + + private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor expected) { + assertEval(opName, x, y, z, q, null, expected, null); + } + + private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r, Tensor expected) { + assertEval(opName, x, y, z, q, r, expected, null); + } + + private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r, Tensor expected, AttributeConverter attr) { Context context = new MapContext(DoubleValue.NaN); - List<IntermediateOperation> inputs = createInputs(context, x, y, z); + List<IntermediateOperation> inputs = createInputs(context, x, y, z, q, r); IntermediateOperation op = mapOperation(opName, inputs, modelName, opName, attr != null ? attr : createAttributes().build()); optimizeAndRename(opName, op); Tensor result = evaluate(op); @@ -363,11 +502,13 @@ public class OnnxOperationsTestCase { assertEquals(expected.type(), result.type()); } - private List<IntermediateOperation> createInputs(Context context, Tensor x, Tensor y, Tensor z) { + private List<IntermediateOperation> createInputs(Context context, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r) { List<IntermediateOperation> inputs = new ArrayList<>(); addInput(inputs, context, x, "x"); addInput(inputs, context, y, "y"); addInput(inputs, context, z, "z"); + addInput(inputs, context, q, "q"); + addInput(inputs, context, r, "r"); return inputs; } @@ -451,6 +592,16 @@ public class OnnxOperationsTestCase { return this; } + Attributes attr(String name, Tensor tensor) { + Onnx.TensorProto.Builder builder = Onnx.TensorProto.newBuilder(); + builder.setDataType(Onnx.TensorProto.DataType.DOUBLE);; + tensor.type().dimensions().forEach(d -> builder.addDims(d.size().get())); + tensor.valueIterator().forEachRemaining(builder::addDoubleData); + Onnx.TensorProto val = builder.build(); + nodeBuilder.addAttribute(Onnx.AttributeProto.newBuilder().setName(name).setType(TENSOR).setT(val).build()); + return this; + } + AttributeConverter build() { return AttributeConverter.convert(nodeBuilder.build()); } diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java index d1dea730da5..9631bddd93d 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java @@ -3,8 +3,13 @@ package ai.vespa.rankingexpression.importer.onnx; import ai.vespa.rankingexpression.importer.ImportedModel; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; @@ -21,21 +26,48 @@ public class SimpleImportTestCase { ImportedModel model = new OnnxImporter().importModel("test", "src/test/models/onnx/simple/simple.onnx"); MapContext context = new MapContext(); - context.put("query_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[4])")). - cell(0.1, 0, 0). - cell(0.2, 0, 1). - cell(0.3, 0, 2). - cell(0.4, 0, 3).build())); - context.put("attribute_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[4],d1[1])")). - cell(0.1, 0, 0). - cell(0.2, 1, 0). - cell(0.3, 2, 0). - cell(0.4, 3, 0).build())); - context.put("bias_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[1])")). - cell(1.0, 0, 0).build())); + context.put("query_tensor", new TensorValue(Tensor.from("tensor(d0[1],d1[4]):[0.1, 0.2, 0.3, 0.4]"))); + context.put("attribute_tensor", new TensorValue(Tensor.from("tensor(d0[4],d1[1]):[0.1, 0.2, 0.3, 0.4]"))); + context.put("bias_tensor", new TensorValue(Tensor.from("tensor(d0[1],d1[1]):[1.0]"))); Tensor result = model.expressions().get("output").evaluate(context).asTensor(); assertEquals(result, Tensor.from("tensor(d0[1],d1[1]):{{d0:0,d1:0}:1.3}")); } + @Test + public void testGather() { + ImportedModel model = new OnnxImporter().importModel("test", "src/test/models/onnx/simple/gather.onnx"); + + MapContext context = new MapContext(); + context.put("data", new TensorValue(Tensor.from("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]"))); + context.put("indices", new TensorValue(Tensor.from("tensor(d0[2],d1[2]):[0, 1, 1, 2]"))); + + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); + + Tensor result = model.expressions().get("y").evaluate(context).asTensor(); + assertEquals(result, Tensor.from("tensor(d0[2],d1[2],d2[2]):[1, 2, 3, 4, 3, 4, 5, 6]")); + } + + private void evaluateFunction(Context context, ImportedModel model, String functionName) { + if (!context.names().contains(functionName)) { + RankingExpression e = RankingExpression.from(model.functions().get(functionName)); + evaluateFunctionDependencies(context, model, e.getRoot()); + context.put(functionName, new TensorValue(e.evaluate(context).asTensor())); + } + } + + private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) { + if (node instanceof ReferenceNode) { + String name = node.toString(); + if (model.functions().containsKey(name)) { + evaluateFunction(context, model, name); + } + } + else if (node instanceof CompositeNode) { + for (ExpressionNode child : ((CompositeNode)node).children()) { + evaluateFunctionDependencies(context, model, child); + } + } + } + } diff --git a/model-integration/src/test/models/lightgbm/classification.json b/model-integration/src/test/models/lightgbm/classification.json new file mode 100644 index 00000000000..1087446519d --- /dev/null +++ b/model-integration/src/test/models/lightgbm/classification.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "binary sigmoid:1", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 13080.099609375, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 8303.599609375, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.598248, + "internal_weight": 14841.2, + "internal_count": 59371, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.10149588882231209, + "leaf_weight": 8812.104370772839, + "leaf_count": 35252 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.05076009488472203, + "leaf_weight": 6029.137221112847, + "leaf_count": 24119 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.1075553310531564, + "leaf_weight": 10156.217760130763, + "leaf_count": 40629 + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 12144.5, + "threshold": 0.4932456977560694, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.07039230856418545, + "leaf_weight": 12362.572675153613, + "leaf_count": 49561 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 6445.509765625, + "threshold": 0.4026061210695467, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.691647, + "internal_weight": 12581.6, + "internal_count": 50439, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.016713933964828474, + "leaf_weight": 5157.183633238077, + "leaf_count": 20675 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.12881836794307533, + "leaf_weight": 7424.385220557451, + "leaf_count": 29764 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 11470.099609375, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.0837843210726433, + "leaf_weight": 9858.360527098179, + "leaf_count": 39612 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 8077.8701171875, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": -0.549408, + "internal_weight": 15039.7, + "internal_count": 60388, + "left_child": { + "leaf_index": 1, + "leaf_value": 0.035561394754096094, + "leaf_weight": 5955.117423638701, + "leaf_count": 23896 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.11424082565448186, + "leaf_weight": 9084.538012728095, + "leaf_count": 36492 + } + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 11022.599609375, + "threshold": 0.5135386524711826, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 5789.919921875, + "threshold": 0.6237474076885036, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.641438, + "internal_weight": 12881.9, + "internal_count": 51907, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.11613056205533928, + "leaf_weight": 8044.6355674266815, + "leaf_count": 32426 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.022313103333779363, + "leaf_weight": 4837.266924858093, + "leaf_count": 19481 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.06927713686880098, + "leaf_weight": 11923.512641906738, + "leaf_count": 48093 + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 9828.9501953125, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.07771712562582928, + "leaf_weight": 9804.427681803703, + "leaf_count": 39586 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 6332.2900390625, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": -0.51112, + "internal_weight": 14922.7, + "internal_count": 60414, + "left_child": { + "leaf_index": 1, + "leaf_value": 0.029062142260340918, + "leaf_weight": 5933.120021238923, + "leaf_count": 23922 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.10400033924773491, + "leaf_weight": 8989.602796778083, + "leaf_count": 36492 + } + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/model-integration/src/test/models/lightgbm/regression.json b/model-integration/src/test/models/lightgbm/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/model-integration/src/test/models/lightgbm/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py new file mode 100755 index 00000000000..ac00437d192 --- /dev/null +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py @@ -0,0 +1,54 @@ +#! /usr/bin/env python3 +# coding: utf-8 + +import json +import random + +import lightgbm as lgb +import numpy as np +import pandas as pd + + +def category_value(arr): + values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 } + return np.array([ 0.21 * values[i] for i in arr ]) + +# Create training set +num_examples = 100000 +missing_prob = 0.01 +features = pd.DataFrame({ + "numerical_1": np.random.random(num_examples), + "numerical_2": np.random.random(num_examples), + "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"), + "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"), + }) + +# randomly insert missing values +for i in range(int(num_examples * len(features.columns) * missing_prob)): + features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None + +# create targets (with 0.0 as default for missing values) +target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"]) +target = (target > 2.24) * 1.0 +lgb_train = lgb.Dataset(features, target) + +# Train model +params = { + 'objective': 'binary', + 'metric': 'binary_logloss', + 'num_leaves': 3, +} +model = lgb.train(params, lgb_train, num_boost_round=5) + +# Save model +with open("classification.json", "w") as f: + json.dump(model.dump_model(), f, indent=2) + +# Predict (for comparison with Vespa evaluation) +predict_features = pd.DataFrame({ + "numerical_1": pd.Series([ None, 0.1, None, 0.7]), + "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]), + "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"), + "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"), + }) +print(model.predict(predict_features)) diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py new file mode 100755 index 00000000000..3e74e38da35 --- /dev/null +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 +# coding: utf-8 + +import json +import random + +import lightgbm as lgb +import numpy as np +import pandas as pd + + +def category_value(arr): + values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 } + return np.array([ 0.21 * values[i] for i in arr ]) + +# Create training set +num_examples = 100000 +missing_prob = 0.01 +features = pd.DataFrame({ + "numerical_1": np.random.random(num_examples), + "numerical_2": np.random.random(num_examples), + "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"), + "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"), + }) + +# randomly insert missing values +for i in range(int(num_examples * len(features.columns) * missing_prob)): + features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None + +# create targets (with 0.0 as default for missing values) +target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"]) +lgb_train = lgb.Dataset(features, target) + +# Train model +params = { + 'objective': 'mse', + 'metric': {'l2', 'l1'}, + 'num_leaves': 3, +} +model = lgb.train(params, lgb_train, num_boost_round=2) + +# Save model +with open("regression.json", "w") as f: + json.dump(model.dump_model(), f, indent=2) + +# Predict (for comparison with Vespa evaluation) +predict_features = pd.DataFrame({ + "numerical_1": pd.Series([ None, 0.1, None, 0.7]), + "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]), + "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"), + "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"), + }) +print(model.predict(predict_features)) diff --git a/model-integration/src/test/models/onnx/simple/gather.onnx b/model-integration/src/test/models/onnx/simple/gather.onnx Binary files differnew file mode 100644 index 00000000000..62451ad953d --- /dev/null +++ b/model-integration/src/test/models/onnx/simple/gather.onnx diff --git a/model-integration/src/test/models/onnx/simple/gather.py b/model-integration/src/test/models/onnx/simple/gather.py new file mode 100755 index 00000000000..63a2103fd86 --- /dev/null +++ b/model-integration/src/test/models/onnx/simple/gather.py @@ -0,0 +1,23 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +import numpy as np +from onnx import helper, TensorProto + +data_type = helper.make_tensor_value_info('data', TensorProto.FLOAT, [3,2]) +indices_type = helper.make_tensor_value_info('indices', TensorProto.FLOAT, [2,2]) +output_type = helper.make_tensor_value_info('y', TensorProto.FLOAT, [2,2,2]) + +node = onnx.helper.make_node( + 'Gather', + inputs=['data', 'indices'], + outputs=['y'], + axis=0, +) +graph_def = onnx.helper.make_graph( + nodes = [node], + name = 'gather_test', + inputs = [data_type, indices_type], + outputs = [output_type] +) +model_def = helper.make_model(graph_def, producer_name='gather.py') +onnx.save(model_def, 'gather.onnx') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java index 233fe8318ae..c1642c71e64 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.host.FlavorOverrides; import java.util.Objects; @@ -19,26 +20,30 @@ public class AddNode { public final Optional<String> nodeFlavor; public final Optional<FlavorOverrides> flavorOverrides; public final Optional<NodeResources> nodeResources; + public final Optional<TenantName> reservedTo; public final NodeType nodeType; public final Set<String> ipAddresses; public final Set<String> additionalIpAddresses; - public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { - return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); + public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<TenantName> reservedTo, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), reservedTo, nodeType, ipAddresses, additionalIpAddresses); } public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { - return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); + return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), Optional.empty(), nodeType, ipAddresses, Set.of()); } private AddNode(String hostname, Optional<String> parentHostname, - Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources, + Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, + Optional<NodeResources> nodeResources, + Optional<TenantName> reservedTo, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { this.hostname = hostname; this.parentHostname = parentHostname; this.nodeFlavor = nodeFlavor; this.flavorOverrides = flavorOverrides; this.nodeResources = nodeResources; + this.reservedTo = reservedTo; this.nodeType = nodeType; this.ipAddresses = ipAddresses; this.additionalIpAddresses = additionalIpAddresses; @@ -52,6 +57,7 @@ public class AddNode { return Objects.equals(hostname, addNode.hostname) && Objects.equals(parentHostname, addNode.parentHostname) && Objects.equals(nodeFlavor, addNode.nodeFlavor) && + Objects.equals(reservedTo, addNode.reservedTo) && nodeType == addNode.nodeType && Objects.equals(ipAddresses, addNode.ipAddresses) && Objects.equals(additionalIpAddresses, addNode.additionalIpAddresses); @@ -59,7 +65,7 @@ public class AddNode { @Override public int hashCode() { - return Objects.hash(hostname, parentHostname, nodeFlavor, nodeType, ipAddresses, additionalIpAddresses); + return Objects.hash(hostname, parentHostname, nodeFlavor, reservedTo, nodeType, ipAddresses, additionalIpAddresses); } @Override @@ -68,9 +74,11 @@ public class AddNode { "hostname='" + hostname + '\'' + ", parentHostname=" + parentHostname + ", nodeFlavor='" + nodeFlavor + '\'' + + (reservedTo.isPresent() ? ", reservedTo='" + reservedTo.get().value() + "'" : "") + ", nodeType=" + nodeType + ", ipAddresses=" + ipAddresses + ", additionalIpAddresses=" + additionalIpAddresses + '}'; } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java index 5317aa74737..a6c876117b9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.TenantName; import java.time.Instant; import java.util.Map; @@ -29,6 +30,7 @@ public class NodeAttributes { private Optional<Version> currentOsVersion = Optional.empty(); private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<Boolean> wantToDeprovision = Optional.empty(); + private Optional<TenantName> reservedTo = Optional.empty(); /** The list of reports to patch. A null value is used to remove the report. */ private Map<String, JsonNode> reports = new TreeMap<>(); @@ -121,10 +123,14 @@ public class NodeAttributes { return reports; } + public Optional<TenantName> getReservedTo() { + return reservedTo; + } + @Override public int hashCode() { return Objects.hash(restartGeneration, rebootGeneration, dockerImage, vespaVersion, currentOsVersion, - currentFirmwareCheck, wantToDeprovision, reports); + currentFirmwareCheck, wantToDeprovision, reports, reservedTo); } public boolean isEmpty() { @@ -145,6 +151,7 @@ public class NodeAttributes { && Objects.equals(currentOsVersion, other.currentOsVersion) && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck) && Objects.equals(reports, other.reports) + && Objects.equals(reservedTo, other.reservedTo) && Objects.equals(wantToDeprovision, other.wantToDeprovision); } @@ -158,6 +165,7 @@ public class NodeAttributes { currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()), currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at), Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports), + Optional.ofNullable(reservedTo.isEmpty() ? null : "reservedTo=" + reservedTo), wantToDeprovision.map(depr -> "wantToDeprovision=" + depr)) .filter(Optional::isPresent) .map(Optional::get) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 38459979e7c..afe092fa325 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -243,6 +244,7 @@ public class RealNodeRepository implements NodeRepository { node.resources.diskSpeed = toString(resources.diskSpeed()); node.resources.storageType = toString(resources.storageType()); }); + node.reservedTo = addNode.reservedTo.map(TenantName::value).orElse(null); node.type = addNode.nodeType.name(); node.ipAddresses = addNode.ipAddresses; node.additionalIpAddresses = addNode.additionalIpAddresses; @@ -258,10 +260,12 @@ public class RealNodeRepository implements NodeRepository { node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null); node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null); node.wantToDeprovision = nodeAttributes.getWantToDeprovision().orElse(null); + node.reservedTo = nodeAttributes.getReservedTo().map(TenantName::value).orElse(null); Map<String, JsonNode> reports = nodeAttributes.getReports(); node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); return node; } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 3fdb8b420ce..4c1f74290e9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -62,6 +62,8 @@ public class NodeRepositoryNode { public Integer failCount; @JsonProperty("environment") public String environment; + @JsonProperty("reservedTo") + public String reservedTo; @JsonProperty("type") public String type; @JsonProperty("wantedDockerImage") @@ -106,6 +108,7 @@ public class NodeRepositoryNode { ", wantedFirmwareCheck=" + wantedFirmwareCheck + ", failCount=" + failCount + ", environment='" + environment + '\'' + + ", reservedTo='" + reservedTo + "'" + ", type='" + type + '\'' + ", wantedDockerImage='" + wantedDockerImage + '\'' + ", currentDockerImage='" + currentDockerImage + '\'' + @@ -188,4 +191,5 @@ public class NodeRepositoryNode { '}'; } } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index ef9e33d20d4..c790e73037e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; @@ -52,13 +50,11 @@ public class DockerOperationsImpl implements DockerOperations { private final Docker docker; private final Terminal terminal; private final IPAddresses ipAddresses; - private final BooleanFlag failStartingNodeOnIpMismatch; public DockerOperationsImpl(Docker docker, Terminal terminal, IPAddresses ipAddresses, FlagSource flagSource) { this.docker = docker; this.terminal = terminal; this.ipAddresses = ipAddresses; - this.failStartingNodeOnIpMismatch = Flags.FAIL_STARTING_NODE_ON_IP_MISMATCH.bindTo(flagSource); } @Override @@ -96,10 +92,8 @@ public class DockerOperationsImpl implements DockerOperations { Optional<? extends InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname()); Optional<? extends InetAddress> ipV6Local = ipAddresses.getIPv6Address(context.node().hostname()); - if (failStartingNodeOnIpMismatch.value()) { - assertEqualIpAddresses(context.hostname(), ipV4Local, context.node().ipAddresses(), IPVersion.IPv4); - assertEqualIpAddresses(context.hostname(), ipV6Local, context.node().ipAddresses(), IPVersion.IPv6); - } + assertEqualIpAddresses(context.hostname(), ipV4Local, context.node().ipAddresses(), IPVersion.IPv4); + assertEqualIpAddresses(context.hostname(), ipV6Local, context.node().ipAddresses(), IPVersion.IPv6); if (ipV4Local.isEmpty() && ipV6Local.isEmpty()) { throw new ConvergenceException("Container " + context.node().hostname() + " with " + networking + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index cb209d710c8..e6d6f9463d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -114,6 +114,10 @@ public class NodeAdminStateUpdater { // To avoid node agents stalling for too long, we'll force unfrozen ticks now. adjustNodeAgentsToRunFromNodeRepository(); nodeAdmin.setFrozen(false); + + NodeState currentNodeState = nodeRepository.getNode(hostHostname).state(); + if (currentNodeState == NodeState.active) orchestrator.resume(hostHostname); + throw new ConvergenceException("Timed out trying to freeze all nodes: will force an unfrozen tick"); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 68178418e62..2f92ef8affe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.log.LogLevel; import com.yahoo.vespa.flags.DoubleFlag; @@ -26,6 +27,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintai import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -47,11 +51,10 @@ public class NodeAgentImpl implements NodeAgent { // This is used as a definition of 1 GB when comparing flavor specs in node-repo private static final long BYTES_IN_GB = 1_000_000_000L; - private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); + // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration + private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(30); - private final AtomicBoolean terminated = new AtomicBoolean(false); - private boolean hasResumedNode = false; - private boolean hasStartedServices = true; + private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); private final NodeAgentContextSupplier contextSupplier; private final NodeRepository nodeRepository; @@ -61,12 +64,19 @@ public class NodeAgentImpl implements NodeAgent { private final Optional<CredentialsMaintainer> credentialsMaintainer; private final Optional<AclMaintainer> aclMaintainer; private final Optional<HealthChecker> healthChecker; + private final Clock clock; + private final Duration warmUpDuration; private final DoubleFlag containerCpuCap; private Thread loopThread; private ContainerState containerState = UNKNOWN; private NodeSpec lastNode; + private final AtomicBoolean terminated = new AtomicBoolean(false); + private boolean hasResumedNode = false; + private boolean hasStartedServices = true; + private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty(); + private int numberOfUnhandledException = 0; private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -87,16 +97,18 @@ public class NodeAgentImpl implements NodeAgent { // Created in NodeAdminImpl - public NodeAgentImpl( - final NodeAgentContextSupplier contextSupplier, - final NodeRepository nodeRepository, - final Orchestrator orchestrator, - final DockerOperations dockerOperations, - final StorageMaintainer storageMaintainer, - final FlagSource flagSource, - final Optional<CredentialsMaintainer> credentialsMaintainer, - final Optional<AclMaintainer> aclMaintainer, - final Optional<HealthChecker> healthChecker) { + public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, + Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer, + FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock) { + this(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, credentialsMaintainer, + aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION); + } + + public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, + Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer, + FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, Duration warmUpDuration) { this.contextSupplier = contextSupplier; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; @@ -105,6 +117,8 @@ public class NodeAgentImpl implements NodeAgent { this.credentialsMaintainer = credentialsMaintainer; this.aclMaintainer = aclMaintainer; this.healthChecker = healthChecker; + this.clock = clock; + this.warmUpDuration = warmUpDuration; this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource); } @@ -194,9 +208,11 @@ public class NodeAgentImpl implements NodeAgent { } } - private void startContainer(NodeAgentContext context) { + private Container startContainer(NodeAgentContext context) { ContainerData containerData = createContainerData(context); - dockerOperations.createContainer(context, containerData, getContainerResources(context)); + ContainerResources wantedResources = context.nodeType() != NodeType.tenant || warmUpDuration.isNegative() ? + getContainerResources(context) : getContainerResources(context).withUnlimitedCpus(); + dockerOperations.createContainer(context, containerData, wantedResources); dockerOperations.startContainer(context); currentRebootGeneration = context.node().wantedRebootGeneration(); @@ -204,6 +220,8 @@ public class NodeAgentImpl implements NodeAgent { hasStartedServices = true; // Automatically started with the container hasResumedNode = false; context.log(logger, "Container successfully started, new containerState is " + containerState); + return dockerOperations.getContainer(context).orElseThrow(() -> + new ConvergenceException("Did not find container that was just started")); } private Optional<Container> removeContainerIfNeededUpdateContainerState( @@ -250,6 +268,7 @@ public class NodeAgentImpl implements NodeAgent { if (containerState == ABSENT) return; try { hasStartedServices = hasResumedNode = false; + firstSuccessfulHealthCheckInstant = Optional.empty(); dockerOperations.stopServices(context); } catch (ContainerNotFoundException e) { containerState = ABSENT; @@ -333,9 +352,15 @@ public class NodeAgentImpl implements NodeAgent { } - private void updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) { + private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) { ContainerResources wantedContainerResources = getContainerResources(context); - if (wantedContainerResources.equalsCpu(existingContainer.resources)) return; + + if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant + .map(clock.instant().minus(warmUpDuration)::isBefore) + .orElse(true)) + return existingContainer; + + if (wantedContainerResources.equalsCpu(existingContainer.resources)) return existingContainer; context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s", wantedContainerResources.toStringCpu(), existingContainer.resources.toStringCpu()); @@ -343,6 +368,8 @@ public class NodeAgentImpl implements NodeAgent { // Only update CPU resources dockerOperations.updateContainer(context, wantedContainerResources.withMemoryBytes(existingContainer.resources.memoryBytes())); + return dockerOperations.getContainer(context).orElseThrow(() -> + new ConvergenceException("Did not find container that was just updated")); } private ContainerResources getContainerResources(NodeAgentContext context) { @@ -430,16 +457,25 @@ public class NodeAgentImpl implements NodeAgent { credentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context)); if (container.isEmpty()) { containerState = STARTING; - startContainer(context); + container = Optional.of(startContainer(context)); containerState = UNKNOWN; } else { - updateContainerIfNeeded(context, container.get()); + container = Optional.of(updateContainerIfNeeded(context, container.get())); } aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); startServicesIfNeeded(context); resumeNodeIfNeeded(context); - healthChecker.ifPresent(checker -> checker.verifyHealth(context)); + if (healthChecker.isPresent()) { + healthChecker.get().verifyHealth(context); + if (firstSuccessfulHealthCheckInstant.isEmpty()) + firstSuccessfulHealthCheckInstant = Optional.of(clock.instant()); + + Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration)); + if (!container.get().resources.equalsCpu(getContainerResources(context))) + throw new ConvergenceException("Refusing to resume until warm up period ends (" + + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); + } // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 4e0fd95384c..eadcd997ae8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -164,6 +164,7 @@ public class RealNodeRepositoryTest { AddNode host = AddNode.forHost("host123.domain.tld", "default", Optional.of(FlavorOverrides.ofDisk(123)), + Optional.empty(), NodeType.confighost, Set.of("::1"), Set.of("::2", "::3")); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 653a6f7f091..ea41ab21b9d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -86,18 +86,19 @@ public class DockerTester implements AutoCloseable { .build(); nodeRepository.updateNodeRepositoryNode(hostSpec); + Clock clock = Clock.systemUTC(); + Metrics metrics = new Metrics(); FileSystem fileSystem = TestFileSystem.create(); DockerOperations dockerOperations = new DockerOperationsImpl(docker, terminal, ipAddresses, flagSource); - Metrics metrics = new Metrics(); NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl( contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, - Optional.empty(), Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); + Optional.empty(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1)); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).pathToContainerStorageFromFileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, - nodeAdmin, HOST_HOSTNAME, Clock.systemUTC()); + nodeAdmin, HOST_HOSTNAME, clock); loopThread = new Thread(() -> { nodeAdminStateUpdater.start(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 9e1f84f07a1..f65ad379a63 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Container; @@ -26,12 +27,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import org.junit.Test; import org.mockito.InOrder; +import java.time.Duration; +import java.time.Instant; import java.util.Optional; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -65,6 +70,7 @@ public class NodeAgentImplTest { private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final ManualClock clock = new ManualClock(Instant.now()); @Test @@ -641,6 +647,86 @@ public class NodeAgentImplTest { verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId())); } + @Test + public void initial_cpu_cap_test() { + NodeSpec.Builder specBuilder = nodeBuilder + .state(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); + + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); + + InOrder inOrder = inOrder(orchestrator, dockerOperations); + + ConvergenceException healthCheckException = new ConvergenceException("Not yet up"); + doThrow(healthCheckException).when(healthChecker).verifyHealth(any()); + for (int i = 0; i < 3; i++) { + try { + nodeAgent.doConverge(context); + fail("Expected to fail with health check exception"); + } catch (ConvergenceException e) { + assertEquals(healthCheckException, e); + } + clock.advance(Duration.ofSeconds(30)); + } + + doNothing().when(healthChecker).verifyHealth(any()); + try { + nodeAgent.doConverge(context); + fail("Expected to fail due to warm up period not yet done"); + } catch (ConvergenceException e) { + assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); + } + inOrder.verify(orchestrator, never()).resume(any()); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(dockerOperations, never()).updateContainer(any(), any()); + + + clock.advance(Duration.ofSeconds(31)); + nodeAgent.doConverge(context); + + inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16))); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(dockerOperations, never()).startContainer(any()); + inOrder.verify(orchestrator).resume(any(String.class)); + + // No changes + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator).resume(any(String.class)); + } + + @Test + public void resumes_normally_if_container_is_already_capped_on_start() { + NodeSpec.Builder specBuilder = nodeBuilder + .state(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); + + InOrder inOrder = inOrder(orchestrator, dockerOperations); + + nodeAgent.doConverge(context); + + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator).resume(any(String.class)); + } + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() .resources(resources) @@ -672,11 +758,28 @@ public class NodeAgentImplTest { } private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { + return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1)); + } + + private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) { mockGetContainer(dockerImage, isRunning); + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(2, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(dockerOperations).createContainer(any(), any(), any()); + + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(1, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(dockerOperations).updateContainer(any(), any()); return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer), - Optional.of(healthChecker)); + Optional.of(healthChecker), clock, warmUpDuration); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index e66fef9ac96..321c5632302 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -41,6 +42,7 @@ public final class Node { private final NodeType type; private final Reports reports; private final Optional<String> modelName; + private final Optional<TenantName> reservedTo; /** Record of the last event of each type happening to this node */ private final History history; @@ -50,52 +52,46 @@ public final class Node { /** Creates a node in the initial state (reserved) */ public static Node createDockerNode(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { - return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), new Flavor(resources), Status.initial(), State.reserved, - Optional.empty(), History.empty(), type, new Reports(), Optional.empty()); + return new Node("fake-" + hostname, new IP.Config(ipAddresses, Set.of()), hostname, Optional.of(parentHostname), + new Flavor(resources), Status.initial(), State.reserved, + Optional.empty(), History.empty(), type, new Reports(), Optional.empty(), Optional.empty()); } /** Creates a node in the initial state (provisioned) */ - public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Optional<String> modelName, Flavor flavor, NodeType type) { + public static Node create(String openStackId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, + Optional<String> modelName, Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, Status.initial(), State.provisioned, - Optional.empty(), History.empty(), type, new Reports(), modelName); + Optional.empty(), History.empty(), type, new Reports(), modelName, reservedTo); } /** Creates a node. See also the {@code create} helper methods. */ public Node(String id, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, - Reports reports, Optional<String> modelName) { - Objects.requireNonNull(id, "A node must have an ID"); - requireNonEmptyString(hostname, "A node must have a hostname"); - Objects.requireNonNull(ipConfig, "A node must a have an IP config"); - requireNonEmptyString(parentHostname, "A parent host name must be a proper value"); - Objects.requireNonNull(flavor, "A node must have a flavor"); - Objects.requireNonNull(status, "A node must have a status"); - Objects.requireNonNull(state, "A null node state is not permitted"); - Objects.requireNonNull(allocation, "A null node allocation is not permitted"); - Objects.requireNonNull(history, "A null node history is not permitted"); - Objects.requireNonNull(type, "A null node type is not permitted"); - Objects.requireNonNull(reports, "A null reports is not permitted"); - Objects.requireNonNull(modelName, "A null modelName is not permitted"); + Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo) { + this.id = Objects.requireNonNull(id, "A node must have an ID"); + this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); + this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); + this.parentHostname = requireNonEmptyString(parentHostname, "A parent host name must be a proper value"); + this.flavor = Objects.requireNonNull(flavor, "A node must have a flavor"); + this.status = Objects.requireNonNull(status, "A node must have a status"); + this.state = Objects.requireNonNull(state, "A null node state is not permitted"); + this.allocation = Objects.requireNonNull(allocation, "A null node allocation is not permitted"); + this.history = Objects.requireNonNull(history, "A null node history is not permitted"); + this.type = Objects.requireNonNull(type, "A null node type is not permitted"); + this.reports = Objects.requireNonNull(reports, "A null reports is not permitted"); + this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); + this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); if (state == State.active) requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address"); + if (parentHostname.isPresent()) { if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); } - this.hostname = hostname; - this.ipConfig = ipConfig; - this.parentHostname = parentHostname; - this.id = id; - this.flavor = flavor; - this.status = status; - this.state = state; - this.allocation = allocation; - this.history = history; - this.type = type; - this.reports = reports; - this.modelName = modelName; + if (type != NodeType.host && reservedTo.isPresent()) + throw new IllegalArgumentException("Only hosts can be reserved to a tenant"); } /** Returns the IP addresses of this node */ @@ -166,6 +162,12 @@ public final class Node { public Optional<String> modelName() { return modelName; } /** + * Returns the tenant this node is reserved to, if any. Only hosts can be reserved to a tenant. + * If this is set, resources on this host cannot be allocated to any other tenant + */ + public Optional<TenantName> reservedTo() { return reservedTo; } + + /** * Returns a copy of this node with wantToRetire set to the given value and updated history. * If given wantToRetire is equal to the current, the method is no-op. */ @@ -209,42 +211,47 @@ public final class Node { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo); } /** Returns a node with the flavor assigned to the given value */ public Node with(Flavor flavor) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with the openStackId set */ public Node withOpenStackId(String openStackId) { - return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(openStackId, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName)); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, Optional.of(modelName), reservedTo); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty()); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, Optional.empty(), reservedTo); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ - public Node downAt(Instant instant) { - return with(history.with(new History.Event(History.Event.Type.down, Agent.system, instant))); + public Node downAt(Instant instant, Agent agent) { + return with(history.with(new History.Event(History.Event.Type.down, agent, instant))); } /** Returns a copy of this with any history record saying it has been detected down removed */ @@ -265,26 +272,39 @@ public final class Node { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName); + Optional.of(allocation), history, type, reports, modelName, reservedTo); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName); + Optional.empty(), history, type, reports, modelName, reservedTo); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName); + allocation, history, type, reports, modelName, reservedTo); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName); + allocation, history, type, reports, modelName, reservedTo); + } + + public Node withReservedTo(TenantName tenant) { + if (type != NodeType.host) + throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, Optional.of(tenant)); + } + + /** Returns a copy of this node which is not reserved to a tenant */ + public Node withoutReservedTo() { + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, Optional.empty()); } /** Returns a copy of this node with the current reboot generation set to the given number at the given instant */ @@ -316,28 +336,32 @@ public final class Node { /** Returns a copy of this node with the given history. */ public Node with(History history) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } public Node with(Reports reports) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, + allocation, history, type, reports, modelName, reservedTo); } - private static void requireNonEmptyString(Optional<String> value, String message) { + private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { Objects.requireNonNull(value, message); value.ifPresent(v -> requireNonEmptyString(v, message)); + return value; } - private static void requireNonEmptyString(String value, String message) { + private static String requireNonEmptyString(String value, String message) { Objects.requireNonNull(value, message); if (value.trim().isEmpty()) throw new IllegalArgumentException(message + ", but was '" + value + "'"); + return value; } - private static void requireNonEmpty(Set<String> values, String message) { - if (values == null || values.isEmpty()) { + private static Set<String> requireNonEmpty(Set<String> values, String message) { + if (values == null || values.isEmpty()) throw new IllegalArgumentException(message); - } + return values; } /** Computes the allocation skew of a host node */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 37987f0512d..bdae658f76e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.Iterator; import java.util.List; @@ -74,11 +75,26 @@ public class NodeList implements Iterable<Node> { !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion())); } + /** Returns the subset of nodes that are currently changing their OS version to given version */ + public NodeList changingOsVersionTo(Version version) { + return filter(node -> node.status().osVersion().changingTo(version)); + } + /** Returns the subset of nodes that are currently changing their OS version */ public NodeList changingOsVersion() { return filter(node -> node.status().osVersion().changing()); } + /** Returns a copy of this sorted by current OS version (lowest to highest) */ + public NodeList byIncreasingOsVersion() { + return nodes.stream() + .sorted(Comparator.comparing(node -> node.status() + .osVersion() + .current() + .orElse(Version.emptyVersion))) + .collect(collectingAndThen(Collectors.toList(), NodeList::wrap)); + } + /** Returns the subset of nodes that are currently on the given OS version */ public NodeList onOsVersion(Version version) { return filter(node -> node.status().osVersion().matches(version)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1c9e1445447..7d925b2a4aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.google.inject.Inject; @@ -10,12 +10,16 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -49,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.BiFunction; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -98,8 +103,8 @@ public class NodeRepository extends AbstractComponent { * This will use the system time to make time-sensitive decisions */ @Inject - public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone) { - this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + public NodeRepository(NodeRepositoryConfig config, NodeFlavors flavors, Curator curator, Zone zone, FlagSource flagSource) { + this(flavors, curator, Clock.systemUTC(), zone, new DnsNameResolver(), DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), flagSource); } /** @@ -107,7 +112,7 @@ public class NodeRepository extends AbstractComponent { * which will be used for time-sensitive decisions. */ public NodeRepository(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, NameResolver nameResolver, - DockerImage dockerImage, boolean useCuratorClientCache) { + DockerImage dockerImage, boolean useCuratorClientCache, FlagSource flagSource) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; @@ -116,7 +121,7 @@ public class NodeRepository extends AbstractComponent { this.osVersions = new OsVersions(this); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); - this.dockerImages = new DockerImages(db, dockerImage); + this.dockerImages = new DockerImages(db, dockerImage, Flags.DOCKER_IMAGE_OVERRIDE.bindTo(flagSource)); this.jobControl = new JobControl(db); // read and write all nodes to make sure they are stored in the latest version of the serialized format @@ -127,8 +132,8 @@ public class NodeRepository extends AbstractComponent { /** Returns the curator database client used by this */ public CuratorDatabaseClient database() { return db; } - /** Returns the Docker image to use for nodes in this */ - public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); } + /** Returns the Docker image to use for given node */ + public DockerImage dockerImage(Node node) { return dockerImages.dockerImageFor(node); } /** @return The name resolver used to resolve hostname and ip addresses */ public NameResolver nameResolver() { return nameResolver; } @@ -193,7 +198,16 @@ public class NodeRepository extends AbstractComponent { /** Returns a filterable list of all load balancers in this repository */ public LoadBalancerList loadBalancers() { - return LoadBalancerList.copyOf(database().readLoadBalancers().values()); + return loadBalancers((ignored) -> true); + } + + /** Returns a filterable list of load balancers belonging to given application */ + public LoadBalancerList loadBalancers(ApplicationId application) { + return loadBalancers((id) -> id.application().equals(application)); + } + + private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) { + return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values()); } public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); } @@ -203,7 +217,7 @@ public class NodeRepository extends AbstractComponent { /** * Returns the ACL for the node (trusted nodes, networks and ports) */ - private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) { + private NodeAcl getNodeAcl(Node node, NodeList candidates) { Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname)); Set<Integer> trustedPorts = new LinkedHashSet<>(); Set<String> trustedNetworks = new LinkedHashSet<>(); @@ -220,10 +234,10 @@ public class NodeRepository extends AbstractComponent { candidates.parentOf(node).ifPresent(trustedNodes::add); node.allocation().ifPresent(allocation -> { trustedNodes.addAll(candidates.owner(allocation.owner()).asList()); - loadBalancers.owner(allocation.owner()).asList().stream() - .map(LoadBalancer::instance) - .map(LoadBalancerInstance::networks) - .forEach(trustedNetworks::addAll); + loadBalancers(allocation.owner()).asList().stream() + .map(LoadBalancer::instance) + .map(LoadBalancerInstance::networks) + .forEach(trustedNetworks::addAll); }); switch (node.type()) { @@ -292,13 +306,12 @@ public class NodeRepository extends AbstractComponent { */ public List<NodeAcl> getNodeAcls(Node node, boolean children) { NodeList candidates = list(); - LoadBalancerList loadBalancers = loadBalancers(); if (children) { return candidates.childrenOf(node).asList().stream() - .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers)) + .map(childNode -> getNodeAcl(childNode, candidates)) .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); } - return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers)); + return Collections.singletonList(getNodeAcl(node, candidates)); } public NodeFlavors getAvailableFlavors() { @@ -309,16 +322,15 @@ public class NodeRepository extends AbstractComponent { /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */ public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname, - Flavor flavor, NodeType type) { + Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { if (ipConfig.primary().isEmpty()) { // TODO: Remove this. Only test code hits this path ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname)); } - return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, type); + return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type); } - public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, - NodeType type) { - return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, type); + public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) { + return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, Optional.empty(), type); } /** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */ @@ -341,7 +353,7 @@ public class NodeRepository extends AbstractComponent { /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */ public List<Node> addNodes(List<Node> nodes) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { var node = nodes.get(i); var message = "Cannot add " + node.hostname() + ": A node with this name already exists"; @@ -361,7 +373,7 @@ public class NodeRepository extends AbstractComponent { /** Sets a list of nodes ready and returns the nodes in the ready state */ public List<Node> setReady(List<Node> nodes, Agent agent, String reason) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> nodesWithResetFields = nodes.stream() .map(node -> { if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty) @@ -585,7 +597,7 @@ public class NodeRepository extends AbstractComponent { } public List<Node> removeRecursively(Node node, boolean force) { - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { List<Node> removed = new ArrayList<>(); if (node.type().isDockerHost()) { @@ -684,7 +696,7 @@ public class NodeRepository extends AbstractComponent { * Writes these nodes after they have changed some internal state but NOT changed their state field. * This does NOT lock the node repository implicitly, but callers are expected to already hold the lock. * - * @param lock Already acquired lock + * @param lock already acquired lock * @return the written nodes for convenience */ public List<Node> write(List<Node> nodes, @SuppressWarnings("unused") Mutex lock) { @@ -713,7 +725,7 @@ public class NodeRepository extends AbstractComponent { // perform operation while holding locks List<Node> resultingNodes = new ArrayList<>(); - try (Mutex lock = lockAllocation()) { + try (Mutex lock = lockUnallocated()) { for (Node node : unallocatedNodes) resultingNodes.add(action.apply(node, lock)); } @@ -738,12 +750,12 @@ public class NodeRepository extends AbstractComponent { /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); } - /** Create a lock which provides exclusive rights to allocating nodes */ - public Mutex lockAllocation() { return db.lockInactive(); } + /** Create a lock which provides exclusive rights to modifying unallocated nodes */ + public Mutex lockUnallocated() { return db.lockInactive(); } /** Acquires the appropriate lock for this node */ public Mutex lock(Node node) { - return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockAllocation(); + return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java index 014d3df8d9a..bad16bf7d12 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java @@ -1,9 +1,6 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.yahoo.config.provision.ApplicationId; - -import java.time.Instant; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> { this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null")); } - /** Returns the subset of load balancers owned by given application */ - public LoadBalancerList owner(ApplicationId application) { - return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application))); - } - /** Returns the subset of load balancers that are in given state */ public LoadBalancerList in(LoadBalancer.State state) { return of(loadBalancers.stream().filter(lb -> lb.state() == state)); } - /** Returns the subset of load balancers that last changed before given instant */ - public LoadBalancerList changedBefore(Instant instant) { - return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant))); - } - public List<LoadBalancer> asList() { return loadBalancers; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index ae2f68a3143..f428e276df8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty"); + nodeRepository.fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 1c27f9d1713..f75621e18a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; @@ -44,13 +45,15 @@ public class DynamicProvisioningMaintainer extends Maintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabled; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, - HostProvisioner hostProvisioner, FlagSource flagSource) { + DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -59,7 +62,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { protected void maintain() { if (! dynamicProvisioningEnabled.value()) return; - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); updateProvisioningNodes(nodes, lock); @@ -68,17 +71,14 @@ public class DynamicProvisioningMaintainer extends Maintainer { } void updateProvisioningNodes(NodeList nodes, Mutex lock) { - Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host) - .asList().stream() - .collect(Collectors.toMap(Node::hostname, Function.identity())); - - Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream() - .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false)) + Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() + .filter(node -> node.parentHostname().isPresent()) .collect(Collectors.groupingBy( - node -> provisionedHostsByHostname.get(node.parentHostname().get()), + node -> node.parentHostname().get(), Collectors.toSet())); - nodesByProvisionedParent.forEach((host, children) -> { + nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); nodeRepository().write(updatedNodes, lock); @@ -112,7 +112,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { NodeResources resources = it.next(); removableHosts.stream() .filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state())) - .filter(host -> host.flavor().resources().satisfies(resources)) + .filter(host -> hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()).satisfies(resources)) .min(Comparator.comparingInt(n -> n.flavor().cost())) .ifPresent(host -> { removableHosts.remove(host); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 438732ad4a8..264df716fa8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -107,7 +107,7 @@ public class FailedExpirer extends Maintainer { .collect(Collectors.toList()); if (unparkedChildren.isEmpty()) { - nodeRepository.park(candidate.hostname(), false, Agent.system, + nodeRepository.park(candidate.hostname(), false, Agent.FailedExpirer, "Parked by FailedExpirer due to hardware issue"); } else { log.info(String.format("Expired failed node %s with hardware issue was not parked because of " + @@ -118,7 +118,7 @@ public class FailedExpirer extends Maintainer { nodesToRecycle.add(candidate); } } - nodeRepository.setDirty(nodesToRecycle, Agent.system, "Expired by FailedExpirer"); + nodeRepository.setDirty(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer"); } /** Returns whether the current node fail count should be used as an indicator of hardware issue */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index a6b88d50acb..21746c96411 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -41,9 +41,9 @@ public class InactiveExpirer extends Expirer { expired.forEach(node -> { if (node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire).get().agent() == Agent.operator) { - nodeRepository.park(node.hostname(), false, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } else { - nodeRepository.setDirty(node, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index bcb0c901f14..e2b70608d58 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -15,6 +15,8 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer { /** Move reserved load balancer that have expired to inactive */ private void expireReserved() { - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(reservedExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.reserved) - .changedBefore(expirationTime); - expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now))); - } + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.reserved, lb -> { + var gracePeriod = now.minus(reservedExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet + db.writeLoadBalancer(lb.with(State.inactive, now)); + }); } /** Deprovision inactive load balancers that have expired */ private void removeInactive() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var now = nodeRepository().clock().instant(); - var expirationTime = now.minus(inactiveExpiry); - var expired = nodeRepository().loadBalancers() - .in(State.inactive) - .changedBefore(expirationTime); - for (var lb : expired) { - if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application - try { - service.remove(lb.id().application(), lb.id().cluster()); - db.removeLoadBalancer(lb.id()); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + var now = nodeRepository().clock().instant(); + withLoadBalancersIn(State.inactive, lb -> { + var gracePeriod = now.minus(inactiveExpiry); + if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet + if (!allocatedNodes(lb.id()).isEmpty()) return; // Still has nodes, do not remove + try { + service.remove(lb.id().application(), lb.id().cluster()); + db.removeLoadBalancer(lb.id()); + } catch (Exception e){ + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", failed.size(), @@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); } } /** Remove reals from inactive load balancers */ private void pruneReals() { var failed = new ArrayList<LoadBalancerId>(); - Exception lastException = null; - try (var lock = db.lockLoadBalancers()) { - var deactivated = nodeRepository().loadBalancers().in(State.inactive); - for (var lb : deactivated) { - var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); - var reals = new LinkedHashSet<>(lb.instance().reals()); - // Remove any real no longer allocated to this application - reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); - try { - service.create(lb.id().application(), lb.id().cluster(), reals, true); - db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); - } catch (Exception e) { - failed.add(lb.id()); - lastException = e; - } + var lastException = new AtomicReference<Exception>(); + withLoadBalancersIn(State.inactive, lb -> { + var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet()); + var reals = new LinkedHashSet<>(lb.instance().reals()); + // Remove any real no longer allocated to this application + reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); + try { + service.create(lb.id().application(), lb.id().cluster(), reals, true); + db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); + } catch (Exception e) { + failed.add(lb.id()); + lastException.set(e); } - } + }); if (!failed.isEmpty()) { log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", failed.size(), @@ -120,7 +113,19 @@ public class LoadBalancerExpirer extends Maintainer { .map(LoadBalancerId::serializedForm) .collect(Collectors.joining(", ")), interval()), - lastException); + lastException.get()); + } + } + + /** Apply operation to all load balancers that exist in given state, while holding lock */ + private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index da0db1f1896..ed6e7cc71ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -15,9 +15,10 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import java.time.Clock; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -35,22 +36,25 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; public class MetricsReporter extends Maintainer { private final Metric metric; - private final Function<HostName, Optional<HostStatus>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final ServiceMonitor serviceMonitor; private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>(); private final Supplier<Integer> pendingRedeploymentsSupplier; + private final Clock clock; MetricsReporter(NodeRepository nodeRepository, Metric metric, Orchestrator orchestrator, ServiceMonitor serviceMonitor, Supplier<Integer> pendingRedeploymentsSupplier, - Duration interval) { + Duration interval, + Clock clock) { super(nodeRepository, interval); this.metric = metric; - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.serviceMonitor = serviceMonitor; this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier; + this.clock = clock; } @Override @@ -125,9 +129,15 @@ public class MetricsReporter extends Maintainer { metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context); - orchestrator.apply(new HostName(node.hostname())) - .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0) - .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context)); + orchestrator.apply(new HostName(node.hostname())).ifPresent(info -> { + int suspended = info.status().isSuspended() ? 1 : 0; + metric.set("suspended", suspended, context); + metric.set("allowedToBeDown", suspended, context); // remove summer 2020. + long suspendedSeconds = info.suspendedSince() + .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds()) + .orElse(0L); + metric.set("suspendedSeconds", suspendedSeconds, context); + }); long numberOfServices; HostName hostName = new HostName(node.hostname()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index dca5c092ad2..ca53b215237 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -97,7 +97,7 @@ public class NodeFailer extends Maintainer { int throttledNodeFailures = 0; // Ready nodes - try (Mutex lock = nodeRepository().lockAllocation()) { + try (Mutex lock = nodeRepository().lockUnallocated()) { updateNodeLivenessEventsForReadyNodes(lock); for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { @@ -265,7 +265,7 @@ public class NodeFailer extends Maintainer { private boolean nodeSuspended(Node node) { try { - return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN; + return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); } catch (HostNameNotFoundException e) { // Treat it as not suspended return false; @@ -323,7 +323,7 @@ public class NodeFailer extends Maintainer { try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock - return nodeRepository().write(node.downAt(clock.instant()), lock); + return nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index db466043d0c..a49049f8b04 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -81,12 +81,12 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry); provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry); nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); - metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval); + metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index 3109e55df5c..e1407f2a41d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -27,7 +27,7 @@ public class ProvisionedExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned"); + nodeRepository.parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index d7dd93522e4..675e3400722 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -115,7 +115,7 @@ public class Rebalancer extends Maintainer { if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false; - nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.system, clock.instant()), lock); + nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.Rebalancer, clock.instant()), lock); return true; } } @@ -154,7 +154,7 @@ public class Rebalancer extends Maintainer { // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host expectedNewNode.flatMap(node -> nodeRepository().getNode(node.hostname(), Node.State.reserved)) - .ifPresent(node -> nodeRepository().setDirty(node, Agent.system, "Expired by Rebalancer")); + .ifPresent(node -> nodeRepository().setDirty(node, Agent.Rebalancer, "Expired by Rebalancer")); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index b0aa389fe7d..03d466dbf09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -28,6 +28,6 @@ public class ReservationExpirer extends Expirer { } @Override - protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.system, "Expired by ReservationExpirer"); } + protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index f46e2f501bc..7522e411e42 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -7,5 +7,15 @@ package com.yahoo.vespa.hosted.provision.node; * @author bratseth */ public enum Agent { - system, application, operator, NodeFailer + operator, // A hosted Vespa operator. Some logic recognizes these events. + application, // An application package change depoyment + system, // An unspecified system agent + // Specific system agents: + NodeFailer, + Rebalancer, + DirtyExpirer, + FailedExpirer, + InactiveExpirer, + ProvisionedExpirer, + ReservationExpirer } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java index b06bbbb54b5..0622862f5ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java @@ -33,6 +33,11 @@ public class OsVersion { return wanted; } + /** Returns whether this node is currently changing its version to the given version */ + public boolean changingTo(Version version) { + return changing() && wanted.get().equals(version); + } + /** Returns whether this node is currently changing its version */ public boolean changing() { return wanted.isPresent() && !current.equals(wanted); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index af32530a156..65cbc17aff5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.node; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.time.Instant; import java.util.Arrays; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java index fd6094ae111..7885cec6b65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java @@ -71,6 +71,6 @@ public class Reports { return this; } - public Reports build() { return new Reports(Collections.unmodifiableMap(reportMap)); } + public Reports build() { return new Reports(reportMap); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 106595fbd47..e10ff3d24cd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -120,9 +120,10 @@ public class OsVersions { /** Trigger upgrade of nodes of given type*/ private void upgrade(NodeType type, Version version) { var nodes = nodeRepository.list().nodeType(type); - var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersion().size()); - var nodesToUpgrade = nodes.not().changingOsVersion() + var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersionTo(version).size()); + var nodesToUpgrade = nodes.not().changingOsVersionTo(version) .not().onOsVersion(version) + .byIncreasingOsVersion() .first(numberToUpgrade); if (nodesToUpgrade.size() == 0) return; log.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + type + " to OS version " + version); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index a28845109dc..f211ea9eac5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -38,6 +38,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -206,7 +207,7 @@ public class CuratorDatabaseClient { toState, toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), - node.type(), node.reports(), node.modelName()); + node.type(), node.reports(), node.modelName(), node.reservedTo()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } @@ -483,18 +484,16 @@ public class CuratorDatabaseClient { // Load balancers public List<LoadBalancerId> readLoadBalancerIds() { - return curatorDatabase.getChildren(loadBalancersRoot).stream() - .map(LoadBalancerId::fromSerializedForm) - .collect(Collectors.toUnmodifiableList()); + return readLoadBalancerIds((ignored) -> true); } - public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() { - return readLoadBalancerIds().stream() - .map(this::readLoadBalancer) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), - Collections::unmodifiableMap)); + public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) { + return readLoadBalancerIds(filter).stream() + .map(this::readLoadBalancer) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()), + Collections::unmodifiableMap)); } public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { @@ -522,14 +521,21 @@ public class CuratorDatabaseClient { transaction.commit(); } - public Lock lockLoadBalancers() { - return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); + public Lock lockLoadBalancers(ApplicationId application) { + return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); } private Path loadBalancerPath(LoadBalancerId id) { return loadBalancersRoot.append(id.serializedForm()); } + private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) { + return curatorDatabase.getChildren(loadBalancersRoot).stream() + .map(LoadBalancerId::fromSerializedForm) + .filter(predicate) + .collect(Collectors.toUnmodifiableList()); + } + private Transaction.Operation createOrSet(Path path, byte[] data) { if (curatorDatabase.exists(path)) { return CuratorOperations.setData(path.getAbsolute(), data); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index ae4c93621e5..66172521d4c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -6,7 +6,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 2cbfbc349a6..9614c1aa5c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -19,7 +19,8 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.Type; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -77,6 +78,7 @@ public class NodeSerializer { private static final String firmwareCheckKey = "firmwareCheck"; private static final String reportsKey = "reports"; private static final String modelNameKey = "modelName"; + private static final String reservedToKey = "reservedTo"; // Node resource fields // ...for hosts and nodes allocated by legacy flavor specs @@ -149,6 +151,7 @@ public class NodeSerializer { node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong(firmwareCheckKey, instant.toEpochMilli())); node.reports().toSlime(object, reportsKey); node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); + node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); } private void toSlime(Flavor flavor, Cursor object) { @@ -222,7 +225,8 @@ public class NodeSerializer { historyFromSlime(object.field(historyKey)), nodeTypeFromString(object.field(nodeTypeKey).asString()), Reports.fromSlime(object.field(reportsKey)), - modelNameFromSlime(object)); + modelNameFromSlime(object), + reservedToFromSlime(object.field(reservedToKey))); } private Status statusFromSlime(Inspector object) { @@ -341,6 +345,13 @@ public class NodeSerializer { return Optional.empty(); } + private Optional<TenantName> reservedToFromSlime(Inspector object) { + if (! object.valid()) return Optional.empty(); + if (object.type() != Type.STRING) + throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); + return Optional.of(TenantName.from(object.asString())); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ @@ -388,19 +399,31 @@ public class NodeSerializer { private Agent eventAgentFromSlime(Inspector eventAgentField) { switch (eventAgentField.asString()) { + case "operator" : return Agent.operator; case "application" : return Agent.application; case "system" : return Agent.system; - case "operator" : return Agent.operator; case "NodeFailer" : return Agent.NodeFailer; + case "Rebalancer" : return Agent.Rebalancer; + case "DirtyExpirer" : return Agent.DirtyExpirer; + case "FailedExpirer" : return Agent.FailedExpirer; + case "InactiveExpirer" : return Agent.InactiveExpirer; + case "ProvisionedExpirer" : return Agent.ProvisionedExpirer; + case "ReservationExpirer" : return Agent.ReservationExpirer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } private String toString(Agent agent) { switch (agent) { + case operator : return "operator"; case application : return "application"; case system : return "system"; - case operator : return "operator"; case NodeFailer : return "NodeFailer"; + case Rebalancer : return "Rebalancer"; + case DirtyExpirer : return "DirtyExpirer"; + case FailedExpirer : return "FailedExpirer"; + case InactiveExpirer : return "InactiveExpirer"; + case ProvisionedExpirer : return "ProvisionedExpirer"; + case ReservationExpirer : return "ReservationExpirer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java index 37dc8a8a1ad..6615dff24e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java index dfa79a4fd9a..e27cdcd1842 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java index 14c8c7fa18e..915b1acedaa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.node.OsVersion; import java.io.IOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java index 1839f7187fa..1149b15a2b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.HashSet; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index a7519de3776..36034b62cfb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; -import com.yahoo.log.LogLevel; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -90,6 +89,22 @@ class Activator { nodeRepository.deactivate(activeToRemove, transaction); nodeRepository.activate(updateFrom(hosts, continuedActive), transaction); // update active with any changes nodeRepository.activate(updatePortsFrom(hosts, reservedToActivate), transaction); + unreserveParentsOf(reservedToActivate); + } + + /** When a tenant node is activated on a host, we can open up that host for use by others */ + private void unreserveParentsOf(List<Node> nodes) { + for (Node node : nodes) { + if ( node.parentHostname().isEmpty()) continue; + Optional<Node> parent = nodeRepository.getNode(node.parentHostname().get()); + if (parent.isEmpty()) continue; + if (parent.get().reservedTo().isEmpty()) continue; + try (Mutex lock = nodeRepository.lock(parent.get())) { + Optional<Node> lockedParent = nodeRepository.getNode(parent.get().hostname()); + if (lockedParent.isEmpty()) continue; + nodeRepository.write(lockedParent.get().withoutReservedTo(), lock); + } + } } /** Activate load balancers */ @@ -117,9 +132,9 @@ class Activator { .filter(node -> node.state() != Node.State.active) .map(Node::hostname) .collect(Collectors.toSet()); - long numNonActive = nonActiveHosts.size(); - if (numNonActive > 0) { - long numActive = parentHostnames.size() - numNonActive; + + if (nonActiveHosts.size() > 0) { + long numActive = parentHostnames.size() - nonActiveHosts.size(); var messageBuilder = new StringBuilder() .append(numActive).append("/").append(parentHostnames.size()) .append(" hosts for ") diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index a609103ac89..fb76dc54d1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -26,12 +26,6 @@ public class DockerHostCapacity { this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } - /** Returns the allocation skew of this host */ - public double skew(Node host) { - NodeResources free = freeCapacityOf(host, false); - return Node.skew(host.flavor().resources(), free); - } - int compareWithoutInactive(Node hostA, Node hostB) { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; @@ -72,7 +66,7 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources()); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index 9a06f2a980a..4416106f23e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -3,9 +3,14 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -13,6 +18,7 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,6 +34,7 @@ public class DockerImages { private final CuratorDatabaseClient db; private final DockerImage defaultImage; private final Duration cacheTtl; + private final StringFlag imageOverride; /** * Docker image is read on every request to /nodes/v2/node/[fqdn]. Cache current getDockerImages to avoid @@ -36,20 +43,41 @@ public class DockerImages { */ private volatile Supplier<Map<NodeType, DockerImage>> dockerImages; - public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage) { - this(db, defaultImage, defaultCacheTtl); + public DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, StringFlag imageOverride) { + this(db, defaultImage, defaultCacheTtl, imageOverride); } - DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl) { + DockerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl, StringFlag imageOverride) { this.db = db; this.defaultImage = defaultImage; this.cacheTtl = cacheTtl; + this.imageOverride = imageOverride; createCache(); } private void createCache() { this.dockerImages = Suppliers.memoizeWithExpiration(() -> Collections.unmodifiableMap(db.readDockerImages()), - cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + cacheTtl.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Returns the image to use for given node and zone */ + public DockerImage dockerImageFor(Node node) { + if (node.type().isDockerHost()) { + // Docker hosts do not run in containers, and thus has no image. Return the image of the child node type + // instead as this allows the host to pre-download the (likely) image its node will run. + // + // Note that if the Docker image has been overridden through feature flag, the preloaded image won't match. + return dockerImageFor(node.type().childNodeType()); + } + return node.allocation() + .map(Allocation::owner) + .map(ApplicationId::serializedForm) + // Return overridden image for this application + .map(application -> imageOverride.with(FetchVector.Dimension.APPLICATION_ID, application).value()) + .filter(Predicate.not(String::isEmpty)) + .map(DockerImage::fromString) + // ... or default Docker image for this node type + .orElseGet(() -> dockerImageFor(node.type())); } /** Returns the current docker images for each node type */ @@ -58,7 +86,7 @@ public class DockerImages { } /** Returns the current docker image for given node type, or default */ - public DockerImage dockerImageFor(NodeType type) { + private DockerImage dockerImageFor(NodeType type) { return getDockerImages().getOrDefault(type, defaultImage); } @@ -69,8 +97,8 @@ public class DockerImages { } try (Lock lock = db.lockDockerImages()) { Map<NodeType, DockerImage> dockerImages = db.readDockerImages(); - - dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), () -> dockerImages.remove(nodeType)); + dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), + () -> dockerImages.remove(nodeType)); db.writeDockerImages(dockerImages); createCache(); // Throw away current cache log.info("Set docker image for " + nodeType + " nodes to " + dockerImage.map(DockerImage::asString).orElse(null)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 05915b82bae..b5c4478cd5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -30,7 +30,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public static class NoopHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index eab8bb68863..5753bbb3c5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -31,7 +31,6 @@ public class GroupPreparer { private final Optional<HostProvisioner> hostProvisioner; private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabledFlag; - private final BooleanFlag enableInPlaceResize; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, @@ -40,7 +39,6 @@ public class GroupPreparer { this.hostProvisioner = hostProvisioner; this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); - this.enableInPlaceResize = Flags.ENABLE_IN_PLACE_RESIZE.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -65,25 +63,23 @@ public class GroupPreparer { boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); - boolean inPlaceResizeEnabled = enableInPlaceResize - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); + boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); try (Mutex lock = nodeRepository.lock(application)) { // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others - try (Mutex allocationLock = nodeRepository.lockAllocation()) { + try (Mutex allocationLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes LockedNodeList nodeList = nodeRepository.list(allocationLock); NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes, spareCount, wantedGroups, nodeRepository.nameResolver(), - hostResourcesCalculator, inPlaceResizeEnabled); + hostResourcesCalculator, allocateFully); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); - prioritizer.addNewDockerNodes(dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty()); + prioritizer.addNewDockerNodes(); // Allocate from the prioritized list NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index c5808a53837..a5570dbf169 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -9,6 +9,6 @@ import com.yahoo.config.provision.NodeResources; public interface HostResourcesCalculator { /** Calculates the resources that are reserved for host level processes and returns the remainder. */ - NodeResources availableCapacityOf(NodeResources hostResources); + NodeResources availableCapacityOf(String flavorName, NodeResources hostResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f0caa358cb8..26f8cffa519 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; @@ -51,8 +51,8 @@ public class LoadBalancerProvisioner { this.db = nodeRepository.database(); this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format - try (var lock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { var loadBalancer = db.readLoadBalancer(id); loadBalancer.ifPresent(db::writeLoadBalancer); } @@ -72,8 +72,9 @@ public class LoadBalancerProvisioner { public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type - try (var loadBalancersLock = db.lockLoadBalancers()) { - provision(application, cluster.id(), false, loadBalancersLock); + if (application.instance().isTester()) return; // Do not provision for tester instances + try (var lock = db.lockLoadBalancers(application)) { + provision(application, cluster.id(), false, lock); } } @@ -89,11 +90,11 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var loadBalancersLock = db.lockLoadBalancers()) { + try (var lock = db.lockLoadBalancers(application)) { var containerClusters = containerClusterOf(clusters); for (var clusterId : containerClusters) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, loadBalancersLock); + provision(application, clusterId, true, lock); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); @@ -107,16 +108,15 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (Mutex loadBalancersLock = db.lockLoadBalancers()) { - deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction); + try (var lock = db.lockLoadBalancers(application)) { + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } } /** Returns load balancers of given application that are no longer referenced by given clusters */ private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) { - var activeLoadBalancersByCluster = nodeRepository.loadBalancers() - .owner(application) + var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application) .in(LoadBalancer.State.active) .asList() .stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 21e2c322594..c92f7889496 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -140,9 +140,9 @@ class NodeAllocation { continue; } node.node = offered.allocate(application, - ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.flavor().resources()), - clock.instant()); + ClusterMembership.from(cluster, highestIndex.add(1)), + requestedNodes.resources().orElse(node.node.flavor().resources()), + clock.instant()); accepted.add(acceptNode(node, false, false)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a17d92117fb..1ef23209369 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -40,29 +40,30 @@ public class NodePrioritizer { private final LockedNodeList allNodes; private final DockerHostCapacity capacity; private final NodeSpec requestedNodes; - private final ApplicationId appId; + private final ApplicationId application; private final ClusterSpec clusterSpec; private final NameResolver nameResolver; private final boolean isDocker; private final boolean isAllocatingForReplacement; private final boolean isTopologyChange; - private final boolean inPlaceResizeEnabled; + /** If set, a host can only have nodes by single tenant and does not allow in-place resizing. */ + private final boolean allocateFully; private final int currentClusterSize; private final Set<Node> spareHosts; - NodePrioritizer(LockedNodeList allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, + NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator, - boolean inPlaceResizeEnabled) { + boolean allocateFully) { this.allNodes = allNodes; this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; - this.appId = appId; + this.application = application; this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, capacity, spares); - this.inPlaceResizeEnabled = inPlaceResizeEnabled; + this.allocateFully = allocateFully; - NodeList nodesInCluster = allNodes.owner(appId).type(clusterSpec.type()).cluster(clusterSpec.id()); + NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); long currentGroups = nonRetiredNodesInCluster.state(Node.State.active).stream() .flatMap(node -> node.allocation() @@ -117,23 +118,19 @@ public class NodePrioritizer { } } - /** - * Add a node on each docker host with enough capacity for the requested flavor - * - * @param exclusively whether the ready docker nodes should only be added on hosts that - * already have nodes allocated to this tenant - */ - void addNewDockerNodes(boolean exclusively) { + /** Add a node on each docker host with enough capacity for the requested flavor */ + void addNewDockerNodes() { if ( ! isDocker) return; LockedNodeList candidates = allNodes - .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state())); + .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state())) + .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant())); - if (exclusively) { + if (allocateFully) { Set<String> candidateHostnames = candidates.asList().stream() .filter(node -> node.type() == NodeType.tenant) .filter(node -> node.allocation() - .map(a -> a.owner().tenant().equals(appId.tenant())) + .map(a -> a.owner().tenant().equals(this.application.tenant())) .orElse(false)) .flatMap(node -> node.parentHostname().stream()) .collect(Collectors.toSet()); @@ -152,7 +149,7 @@ public class NodePrioritizer { if (host.status().wantToRetire()) continue; boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources); - boolean conflictingCluster = allNodes.childrenOf(host).owner(appId).asList().stream() + boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream() .anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id())); if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; @@ -188,7 +185,7 @@ public class NodePrioritizer { .filter(node -> node.type() == requestedNodes.type()) .filter(node -> legalStates.contains(node.state())) .filter(node -> node.allocation().isPresent()) - .filter(node -> node.allocation().get().owner().equals(appId)) + .filter(node -> node.allocation().get().owner().equals(application)) .map(node -> toPrioritizable(node, false, false)) .forEach(prioritizableNode -> nodes.put(prioritizableNode.node, prioritizableNode)); } @@ -217,7 +214,7 @@ public class NodePrioritizer { builder.parent(parent).freeParentCapacity(parentCapacity); if (!isNewNode) - builder.resizable(inPlaceResizeEnabled && requestedNodes.canResize( + builder.resizable(!allocateFully && requestedNodes.canResize( node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize)); if (spareHosts.contains(parent)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 7cf55adc776..07cf297a314 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -131,7 +131,8 @@ public class NodeRepositoryProvisioner implements Provisioner { private boolean hasQuota(ApplicationId application, int requestedNodes) { if ( ! this.zone.system().isPublic()) return true; // no quota management - if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60; + if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60; + if (application.tenant().value().hashCode() == -1271827001) return requestedNodes <= 75; return requestedNodes <= 5; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index c7732695069..6183bffe5ba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -87,6 +87,10 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { throw new IllegalStateException("Nodes " + this.node + " and " + other.node + " have different states"); if (this.parent.isPresent() && other.parent.isPresent()) { + // Prefer reserved hosts (that they are reserved to the right tenant is ensured elsewhere) + if ( this.parent.get().reservedTo().isPresent() && ! other.parent.get().reservedTo().isPresent()) return -1; + if ( ! this.parent.get().reservedTo().isPresent() && other.parent.get().reservedTo().isPresent()) return 1; + int diskCostDifference = NodeResources.DiskSpeed.compare(this.parent.get().flavor().resources().diskSpeed(), other.parent.get().flavor().resources().diskSpeed()); if (diskCostDifference != 0) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 451ab089a8d..b979ccda740 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -34,7 +34,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, NodeType.host); + return Node.create(id, IP.Config.EMPTY, hostHostname, Optional.empty(), Optional.empty(), hostFlavor, Optional.empty(), NodeType.host); } /** Generate {@link Node} instance representing the node running on this physical host */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index 9f8f4a804d1..3147d4caded 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.ApplicationId; @@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse { } private List<LoadBalancer> loadBalancers() { - LoadBalancerList loadBalancers = nodeRepository.loadBalancers(); - return application().map(loadBalancers::owner) - .map(LoadBalancerList::asList) - .orElseGet(loadBalancers::asList); + LoadBalancerList loadBalancers; + var application = application(); + if (application.isPresent()) { + loadBalancers = nodeRepository.loadBalancers(application.get()); + } else { + loadBalancers = nodeRepository.loadBalancers(); + } + return loadBalancers.asList(); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 81cf401d358..11be95604c8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -6,11 +6,13 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -33,8 +35,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; -import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static com.yahoo.config.provision.NodeResources.StorageType.local; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; /** * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. @@ -45,7 +47,6 @@ import static com.yahoo.config.provision.NodeResources.StorageType.local; public class NodePatcher { private static final String WANT_TO_RETIRE = "wantToRetire"; - private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -99,7 +100,6 @@ public class NodePatcher { private List<Node> applyFieldRecursive(List<Node> childNodes, String name, Inspector value) { switch (name) { case WANT_TO_RETIRE: - case WANT_TO_DEPROVISION: return childNodes.stream() .map(child -> applyField(child, name, value)) .collect(Collectors.toList()); @@ -138,7 +138,9 @@ public class NodePatcher { return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes); case WANT_TO_RETIRE : return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case WANT_TO_DEPROVISION : + case "wantToDeprovision" : + if (node.type() != NodeType.host && asBoolean(value)) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); return node.with(node.status().withWantToDeprovision(asBoolean(value))); case "reports" : return nodeWithPatchedReports(node, value); @@ -160,34 +162,53 @@ public class NodePatcher { case "bandwidthGbps": return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); case "modelName": - if (value.type() == Type.NIX) { - return node.withoutModelName(); - } - return node.withModelName(asString(value)); + return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value)); case "requiredDiskSpeed": return patchRequiredDiskSpeed(asString(value)); + case "reservedTo": + return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); default : throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } } private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { + Node patchedNode; // "reports": null clears the reports - if (reportsInspector.type() == Type.NIX) return node.with(new Reports()); + if (reportsInspector.type() == Type.NIX) { + patchedNode = node.with(new Reports()); + } else { + var reportsBuilder = new Reports.Builder(node.reports()); + reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { + if (reportInspector.type() == Type.NIX) { + // ... "reports": { "reportId": null } clears the report "reportId" + reportsBuilder.clearReport(reportId); + } else { + // ... "reports": { "reportId": {...} } overrides the whole report "reportId" + reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); + } + }); + patchedNode = node.with(reportsBuilder.build()); + } - var reportsBuilder = new Reports.Builder(node.reports()); + boolean hadHardFailReports = node.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); + boolean hasHardFailReports = patchedNode.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); - reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { - if (reportInspector.type() == Type.NIX) { - // ... "reports": { "reportId": null } clears the report "reportId" - reportsBuilder.clearReport(reportId); - } else { - // ... "reports": { "reportId": {...} } overrides the whole report "reportId" - reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); - } - }); + // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa + if (hadHardFailReports != hasHardFailReports) { + // Do not automatically change wantToDeprovision when + // 1. Transitioning to having a HARD_FAIL report and being in state failed: + // To allow operators manually unset before the host is parked and deleted. + // 2. When in parked state: Deletion is imminent, possibly already underway + if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) + return patchedNode; + + patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + } - return node.with(reportsBuilder.build()); + return patchedNode; } private Set<String> asStringSet(Inspector field) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 692402ea914..f0d996ef595 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -19,7 +20,8 @@ import com.yahoo.restapi.ResourceResponse; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.Type; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -49,7 +51,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; -import static com.yahoo.vespa.config.SlimeUtils.optionalString; +import static com.yahoo.slime.SlimeUtils.optionalString; /** * The implementation of the /nodes/v2 API. @@ -225,14 +227,15 @@ public class NodesApiHandler extends LoggingRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); - return Node.create( - inspector.field("openStackId").asString(), - new IP.Config(ipAddresses, ipAddressPool), - inspector.field("hostname").asString(), - parentHostname, - modelName, - flavorFromSlime(inspector), - nodeTypeFromSlime(inspector.field("type"))); + return Node.create(inspector.field("openStackId").asString(), + new IP.Config(ipAddresses, ipAddressPool), + inspector.field("hostname").asString(), + parentHostname, + modelName, + flavorFromSlime(inspector), + reservedToFromSlime(inspector.field("reservedTo")), + nodeTypeFromSlime(inspector.field("type")) + ); } private Flavor flavorFromSlime(Inspector inspector) { @@ -277,6 +280,13 @@ public class NodesApiHandler extends LoggingRequestHandler { return serializer.typeFrom(object.asString()); } + private Optional<TenantName> reservedToFromSlime(Inspector object) { + if (! object.valid()) return Optional.empty(); + if ( object.type() != Type.STRING) + throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); + return Optional.of(TenantName.from(object.asString())); + } + public static NodeFilter toNodeFilter(HttpRequest request) { NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), request.getProperty("flavor"), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 7ed05432e01..7f283452538 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -16,11 +16,10 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.io.IOException; import java.io.OutputStream; @@ -46,7 +45,7 @@ class NodesResponse extends HttpResponse { private final NodeFilter filter; private final boolean recursive; - private final Function<HostName, Optional<HostStatus>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; private final Slime slime; private final NodeSerializer serializer = new NodeSerializer(); @@ -58,7 +57,7 @@ class NodesResponse extends HttpResponse { this.nodeParentUrl = toNodeParentUrl(request); filter = NodesApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; slime = new Slime(); @@ -146,6 +145,7 @@ class NodesResponse extends HttpResponse { } object.setString("openStackId", node.id()); object.setString("flavor", node.flavor().name()); + node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) object.setDouble("cpuCores", node.flavor().getMinCpuCores()); toSlime(node.flavor().resources(), object.setObject("resources")); @@ -157,13 +157,15 @@ class NodesResponse extends HttpResponse { toSlime(allocation.membership(), object.setObject("membership")); object.setLong("restartGeneration", allocation.restartGeneration().wanted()); object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); - object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString()); + object.setString("wantedDockerImage", nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString()); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) - .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN) - .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown)); + .ifPresent(info -> { + object.setBool("allowedToBeDown", info.status().isSuspended()); + info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli())); + }); }); object.setLong("rebootGeneration", node.status().reboot().wanted()); object.setLong("currentRebootGeneration", node.status().reboot().current()); @@ -220,16 +222,10 @@ class NodesResponse extends HttpResponse { // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { return node.status().dockerImage() - .or(() -> Optional.of(node) - .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .flatMap(n -> n.status().vespaVersion() - .map(version -> dockerImageFor(n.type()).withTag(version)))); - } - - // Docker hosts are not running in an image, but return the image of the node type running on it anyway, - // this allows the docker host to pre-download the (likely) image its node will run - private DockerImage dockerImageFor(NodeType nodeType) { - return nodeRepository.dockerImage(nodeType.isDockerHost() ? nodeType.childNodeType() : nodeType); + .or(() -> Optional.of(node) + .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + .flatMap(n -> n.status().vespaVersion() + .map(version -> nodeRepository.dockerImage(n).withTag(version)))); } private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 4abf6d77268..d26accd7a84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -13,7 +13,8 @@ public class ContainerConfig { return "<container version='1.0'>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>20</maxthreads>\n" + - " </config> \n" + + " </config>\n" + + " <accesslog type='disabled'/>\n" + " <component id='com.yahoo.test.ManualClock'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0d331ac0f3f..a2579bee0a1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -54,7 +54,7 @@ public class MockNodeRepository extends NodeRepository { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); @@ -70,34 +70,34 @@ public class MockNodeRepository extends NodeRepository { // Regular nodes nodes.add(createNode("node1", "host1.yahoo.com", ipConfig(1), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); nodes.add(createNode("node2", "host2.yahoo.com", ipConfig(2), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); nodes.add(createNode("node3", "host3.yahoo.com", ipConfig(3), Optional.empty(), - new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); Node node4 = createNode("node4", "host4.yahoo.com", ipConfig(4), Optional.of("dockerhost1.yahoo.com"), - new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), Optional.empty(), NodeType.tenant); node4 = node4.with(node4.status() .withVespaVersion(new Version("6.41.0")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.41.0"))); nodes.add(node4); Node node5 = createNode("node5", "host5.yahoo.com", ipConfig(5), Optional.of("dockerhost2.yahoo.com"), - new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), NodeType.tenant); + new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), Optional.empty(), NodeType.tenant); nodes.add(node5.with(node5.status() .withVespaVersion(new Version("1.2.3")) .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:1.2.3")))); nodes.add(createNode("node6", "host6.yahoo.com", ipConfig(6), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant)); Node node7 = createNode("node7", "host7.yahoo.com", ipConfig(7), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); nodes.add(node7); // 8, 9, 11 and 12 are added by web service calls Node node10 = createNode("node10", "host10.yahoo.com", ipConfig(10), Optional.of("parent1.yahoo.com"), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); Status node10newStatus = node10.status(); node10newStatus = node10newStatus .withVespaVersion(Version.fromString("5.104.142")) @@ -106,26 +106,26 @@ public class MockNodeRepository extends NodeRepository { nodes.add(node10); Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(), - new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant); + new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); /* Setup docker hosts (two of these will be reserved for spares */ nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipConfig(101, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipConfig(102, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipConfig(103, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipConfig(104, 1, 3), Optional.empty(), - flavors.getFlavorOrThrow("large"), NodeType.host)); + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); // Config servers nodes.add(createNode("cfg1", "cfg1.yahoo.com", ipConfig(201), Optional.empty(), - flavors.getFlavorOrThrow("default"), NodeType.config)); + flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config)); nodes.add(createNode("cfg2", "cfg2.yahoo.com", ipConfig(202), Optional.empty(), - flavors.getFlavorOrThrow("default"), NodeType.config)); + flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config)); // Ready all nodes, except 7 and 55 nodes = addNodes(nodes); @@ -167,9 +167,9 @@ public class MockNodeRepository extends NodeRepository { List<Node> largeNodes = new ArrayList<>(); largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(), - new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); largeNodes.add(createNode("node14", "host14.yahoo.com", ipConfig(14), Optional.empty(), - new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant)); + new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant)); addNodes(largeNodes); setReady(largeNodes, Agent.system, getClass().getSimpleName()); ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index 96ec0349fb2..d183e62c96c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -1,16 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; +import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -18,9 +23,9 @@ import java.util.function.Function; /** * @author bratseth */ -public class OrchestratorMock implements Orchestrator { +public class OrchestratorMock extends AbstractComponent implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -30,12 +35,13 @@ public class OrchestratorMock implements Orchestrator { @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = suspendedHosts.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostStatus>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)); + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -48,7 +54,7 @@ public class OrchestratorMock implements Orchestrator { @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override @@ -73,7 +79,7 @@ public class OrchestratorMock implements Orchestrator { } @Override - public void acquirePermissionToRemove(HostName hostName) {} + public void acquirePermissionToRemove(HostName hostName) { } @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index f14b6d59ee0..95555185292 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -27,9 +28,12 @@ public class NodeRepositoryTester { private final NodeRepository nodeRepository; private final Clock clock; private final MockCurator curator; - - + public NodeRepositoryTester() { + this(new InMemoryFlagSource()); + } + + public NodeRepositoryTester(InMemoryFlagSource flagSource) { nodeFlavors = new NodeFlavors(createConfig()); clock = new ManualClock(); curator = new MockCurator(); @@ -37,7 +41,7 @@ public class NodeRepositoryTester { nodeRepository = new NodeRepository(nodeFlavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, flagSource); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index da174a8d38c..96236b5fb84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -22,6 +22,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; @@ -54,7 +55,8 @@ public class CapacityCheckerTester { Curator curator = new MockCurator(); NodeFlavors f = new NodeFlavors(new FlavorConfigBuilder().build()); nodeRepository = new NodeRepository(f, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); } private void updateCapacityChecker() { @@ -133,8 +135,8 @@ public class CapacityCheckerTester { NodeResources nr = containingNodeResources(childResources, excessCapacity); Node node = nodeRepository.createNode(hostname, hostname, - new IP.Config(Set.of("::"), availableIps), Optional.empty(), - new Flavor(nr), NodeType.host); + new IP.Config(Set.of("::"), availableIps), Optional.empty(), + new Flavor(nr), Optional.empty(), NodeType.host); hosts.add(node); } return hosts; @@ -152,8 +154,8 @@ public class CapacityCheckerTester { .mapToObj(n -> String.format("%04X::%04X", hostid, n)) .collect(Collectors.toSet()); Node node = nodeRepository.createNode(hostname, hostname, - new IP.Config(Set.of("::"), availableIps), Optional.empty(), - new Flavor(capacity), NodeType.host); + new IP.Config(Set.of("::"), availableIps), Optional.empty(), + new Flavor(capacity), Optional.empty(), NodeType.host); hosts.add(node); } return hosts; @@ -263,8 +265,8 @@ public class CapacityCheckerTester { Flavor f = new Flavor(nr); Node node = nodeRepository.createNode(nodeModel.id, nodeModel.hostname, - new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses), - nodeModel.parentHostname, f, nodeModel.type); + new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses), + nodeModel.parentHostname, f, Optional.empty(), nodeModel.type); if (membership != null) { return node.allocate(owner, membership, node.flavor().resources(), Instant.now()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 71788fb1a30..36f876fb6bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -49,6 +51,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -64,18 +67,23 @@ public class DynamicProvisioningMaintainerTest { private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true) .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer( - tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource); + tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource); @Test public void delegates_to_host_provisioner_and_writes_back_result() { addNodes(); + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + + Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); + when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); @@ -83,8 +91,10 @@ public class DynamicProvisioningMaintainerTest { maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); + verify(hostProvisioner).provision(eq(host3), eq(Set.of())); verifyNoMoreInteractions(hostProvisioner); + assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); } @@ -127,7 +137,7 @@ public class DynamicProvisioningMaintainerTest { @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); + flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(2, 4, 8, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); addNodes(); maintainer.convergeToCapacity(tester.nodeRepository.list()); @@ -150,6 +160,15 @@ public class DynamicProvisioningMaintainerTest { verifyNoMoreInteractions(hostProvisioner); } + @Before + public void setup() { + doAnswer(invocation -> { + String flavorName = invocation.getArgument(0, String.class); + if ("default".equals(flavorName)) return new NodeResources(2, 4, 8, 1); + return invocation.getArguments()[1]; + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); + } + public void addNodes() { List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), @@ -157,6 +176,7 @@ public class DynamicProvisioningMaintainerTest { createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), @@ -186,7 +206,8 @@ public class DynamicProvisioningMaintainerTest { private final ManualClock clock = new ManualClock(); private final NodeRepository nodeRepository = new NodeRepository( - nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true); + nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-image"), true, new InMemoryFlagSource()); Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { Node node = createNode(hostname, parentHostname, nodeType, state, application); @@ -204,7 +225,7 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); } } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 8509722b016..c293a3436b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -256,7 +256,7 @@ public class FailedExpirerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), - true); + true, new InMemoryFlagSource()); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 12b48fd7a35..7e8fcddb1ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Vtag; @@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue; */ public class LoadBalancerExpirerTest { - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void expire_inactive() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Deploy two applications with a total of three load balancers ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs"); @@ -67,14 +67,15 @@ public class LoadBalancerExpirerTest { // Expirer prunes reals before expiration time of load balancer itself expirer.maintain(); assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals()); - assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals()); + assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals()); // Expirer defers removal of load balancer until expiration time passes expirer.maintain(); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state()); assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1)); // Expirer removes load balancers once expiration time passes - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1)); @@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest { // A single cluster is removed deployApplication(app2, cluster1); expirer.maintain(); - assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); + assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state()); // Expirer defers removal while nodes are still allocated to cluster expirer.maintain(); @@ -93,7 +94,7 @@ public class LoadBalancerExpirerTest { dirtyNodesOf(app2, cluster2); // Expirer removes load balancer for removed cluster - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb3)); } @@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), tester.loadBalancerService()); - Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); + Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Prepare application @@ -121,7 +122,7 @@ public class LoadBalancerExpirerTest { // Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout dirtyNodesOf(app, cluster); - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); @@ -130,7 +131,7 @@ public class LoadBalancerExpirerTest { assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state()); // Expirer removes inactive load balancer - tester.clock().advance(Duration.ofHours(1)); + tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1))); expirer.maintain(); assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index a4b66d3cf9e..246f2509397 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -35,7 +36,7 @@ public class MaintenanceTester { public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); public MaintenanceTester() { curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 539d8c7cff2..321812497bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -10,8 +10,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -22,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; @@ -29,6 +32,7 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -53,7 +57,7 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node)); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); @@ -82,12 +86,17 @@ public class MetricsReporterTest { expectedMetrics.put("wantToRetire", 0); expectedMetrics.put("wantToDeprovision", 0); expectedMetrics.put("failReport", 0); - expectedMetrics.put("allowedToBeDown", 0); + expectedMetrics.put("allowedToBeDown", 1); + expectedMetrics.put("suspended", 1); + expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); + ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); + when(orchestrator.getHostResolver()).thenReturn(hostName -> + Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))) + ); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); @@ -99,8 +108,8 @@ public class MetricsReporterTest { orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1) - ); + Duration.ofMinutes(1), + clock); metricsReporter.maintain(); assertEquals(expectedMetrics, metric.values); @@ -113,13 +122,13 @@ public class MetricsReporterTest { NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", - Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); nodeRepository.addNodes(List.of(dockerHost)); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); @@ -127,29 +136,30 @@ public class MetricsReporterTest { Node container1 = Node.createDockerNode(Set.of("::2"), "container1", "dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1"), container1).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated())); Node container2 = Node.createDockerNode(Set.of("::3"), "container2", "dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2"), container2).get()); - nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockAllocation())); + nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated())); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS)); + when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); + ManualClock clock = new ManualClock(); MetricsReporter metricsReporter = new MetricsReporter( nodeRepository, metric, orchestrator, serviceMonitor, () -> 42, - Duration.ofMinutes(1) - ); + Duration.ofMinutes(1), + clock); metricsReporter.maintain(); assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 5872a78e1e2..033ddcd827e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -75,7 +75,8 @@ public class NodeFailTester { clock = new ManualClock(); curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, + new InMemoryFlagSource()); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 50c00c730bb..22d7f03c449 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -56,7 +56,7 @@ public class OperatorChangeApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, this.fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index b1c3b23016c..913b8b53c46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -62,7 +62,7 @@ public class PeriodicApplicationMaintainerTest { this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d1a330a3bd6..d0c678bdf45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -152,7 +152,7 @@ public class RebalancerTest { private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 11ee6637720..96c3cc09b6b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -47,7 +47,7 @@ public class ReservationExpirerTest { NodeRepository nodeRepository = new NodeRepository(flavors, curator, clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, new InMemoryFlagSource()); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 67af2df36e7..bdbe046fbdf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -64,8 +64,8 @@ public class RetiredExpirerTest { private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, new InMemoryFlagSource()); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index 5e44b2e903e..fcf2ab5a52d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -179,7 +179,7 @@ public class IPTest { private static Node createNode(Set<String> ipAddresses) { return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - NodeType.host); + Optional.empty(), NodeType.host); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index ebb64d650f1..5e859cd3d25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -5,13 +5,18 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.OsVersion; +import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -75,8 +80,11 @@ public class OsVersionsTest { tester.makeReadyNodes(totalNodes, "default", NodeType.host); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host); - // Some nodes have reported current version - setCurrentVersion(hostNodes.get().asList().subList(0, 2), Version.fromString("7.0")); + // 5 nodes have no version. The other 15 are spread across different versions + var hostNodesList = hostNodes.get().asList(); + for (int i = totalNodes - maxActiveUpgrades - 1; i >= 0; i--) { + setCurrentVersion(List.of(hostNodesList.get(i)), new Version(7, 0, i)); + } // Set target var version1 = Version.fromString("7.1"); @@ -86,28 +94,68 @@ public class OsVersionsTest { // Activate target for (int i = 0; i < totalNodes; i += maxActiveUpgrades) { versions.setActive(NodeType.host, true); - var nodesUpgrading = hostNodes.get().changingOsVersion(); + var nodes = hostNodes.get(); + var nodesUpgrading = nodes.changingOsVersion(); assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size()); assertEquals("Wanted version is set for nodes upgrading", version1, - nodesUpgrading.stream() - .map(node -> node.status().osVersion().wanted().get()) - .min(Comparator.naturalOrder()).get()); + minVersion(nodesUpgrading, OsVersion::wanted)); + var nodesOnLowestVersion = nodes.asList().stream() + .sorted(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion))) + .collect(Collectors.toList()) + .subList(0, maxActiveUpgrades); + assertEquals("Nodes on lowest version are told to upgrade", + nodesUpgrading.asList(), nodesOnLowestVersion); completeUpgradeOf(nodesUpgrading.asList()); } // Activating again after all nodes have upgraded does nothing versions.setActive(NodeType.host, true); - assertEquals(version1, hostNodes.get().stream() - .map(n -> n.status().osVersion().current().get()) - .min(Comparator.naturalOrder()).get()); + assertEquals("All nodes upgraded", version1, minVersion(hostNodes.get(), OsVersion::current)); + } + + @Test + public void test_newer_upgrade_aborts_upgrade_to_stale_version() { + var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE); + tester.makeReadyNodes(10, "default", NodeType.host); + Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host); + + // Some nodes are targeting an older version + var version1 = Version.fromString("7.1"); + setWantedVersion(hostNodes.get().asList().subList(0, 5), version1); + + // Trigger upgrade to next version + var version2 = Version.fromString("7.2"); + versions.setTarget(NodeType.host, version2, false); + versions.setActive(NodeType.host, true); + + // Wanted version is changed to newest target for all nodes + assertEquals(version2, minVersion(hostNodes.get(), OsVersion::wanted)); + } + + private Version minVersion(NodeList nodes, Function<OsVersion, Optional<Version>> versionField) { + return nodes.asList().stream() + .map(Node::status) + .map(Status::osVersion) + .map(versionField) + .flatMap(Optional::stream) + .min(Comparator.naturalOrder()) + .orElse(Version.emptyVersion); + + } + + private void setWantedVersion(List<Node> nodes, Version wantedVersion) { + writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion))))); } private void setCurrentVersion(List<Node> nodes, Version currentVersion) { + writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))))); + } + + private void writeNode(List<Node> nodes, UnaryOperator<Node> updateFunc) { for (var node : nodes) { try (var lock = tester.nodeRepository().lock(node)) { node = tester.nodeRepository().getNode(node.hostname()).get(); - node = node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion)))); - tester.nodeRepository().write(node, lock); + tester.nodeRepository().write(updateFunc.apply(node), lock); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 08e7772b5ba..8e048001b98 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -229,7 +229,7 @@ public class SerializationTest { @Test public void serialize_parentHostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", Optional.of(parentHostname), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.tenant); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); assertEquals(parentHostname, deserializedNode.parentHostname().get()); @@ -441,7 +441,8 @@ public class SerializationTest { Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), - NodeType.tenant); + Optional.empty(), NodeType.tenant + ); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 92d066e5f16..1d028f13340 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -40,14 +38,14 @@ public class AclProvisioningTest { tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 1)); List<Node> dockerHost = tester.makeReadyNodes(1, new NodeResources(1, 4, 10, 1), NodeType.host); ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host)); tester.makeReadyVirtualDockerNodes(1,new NodeResources(1, 4, 10, 1), dockerHost.get(0).hostname()); List<Node> proxyNodes = tester.makeReadyNodes(3, new NodeResources(1, 4, 10, 1), NodeType.proxy); // Allocate 2 nodes ApplicationId application = tester.makeApplicationId(); - List<Node> activeNodes = deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); + List<Node> activeNodes = tester.deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true)); assertEquals(2, activeNodes.size()); // Get trusted nodes for the first active node @@ -112,7 +110,7 @@ public class AclProvisioningTest { // Deploy zone application ApplicationId zoneApplication = tester.makeApplicationId(); - deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); + tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy)); // Get trusted nodes for first proxy node List<Node> proxyNodes = tester.nodeRepository().getNodes(zoneApplication); @@ -154,7 +152,7 @@ public class AclProvisioningTest { // Allocate ApplicationId controllerApplication = tester.makeApplicationId(); - List<Node> controllers = deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); + List<Node> controllers = tester.deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller)); // Controllers and hosts all trust each other List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false); @@ -164,12 +162,33 @@ public class AclProvisioningTest { @Test public void trusted_nodes_for_application_with_load_balancer() { - // Populate repo - tester.makeReadyNodes(10, nodeResources); + // Provision hosts and containers + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1), + host.hostname()); + } - // Allocate 2 nodes - List<Node> activeNodes = deploy(2); + // Deploy application + var application = tester.makeApplicationId(); + List<Node> activeNodes = deploy(application, 2); assertEquals(2, activeNodes.size()); + + // Load balancer is allocated to application + var loadBalancers = tester.nodeRepository().loadBalancers(application); + assertEquals(1, loadBalancers.asList().size()); + var lbNetworks = loadBalancers.asList().get(0).instance().networks(); + assertEquals(2, lbNetworks.size()); + + // ACL for nodes with allocation trust their respective load balancer networks, if any + for (var host : hosts) { + var acls = tester.nodeRepository().getNodeAcls(host, true); + assertEquals(2, acls.size()); + assertEquals(Set.of(), acls.get(0).trustedNetworks()); + assertEquals(application, acls.get(1).node().allocation().get().owner()); + assertEquals(lbNetworks, acls.get(1).trustedNetworks()); + } } @Test @@ -191,15 +210,7 @@ public class AclProvisioningTest { } private List<Node> deploy(ApplicationId application, int nodeCount) { - return deploy(application, Capacity.fromCount(nodeCount, nodeResources)); - } - - private List<Node> deploy(ApplicationId application, Capacity capacity) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false); - List<HostSpec> prepared = tester.prepare(application, cluster, capacity, 1); - tester.activate(application, Set.copyOf(prepared)); - return tester.getNodes(application, Node.State.active).asList(); + return tester.deploy(application, Capacity.fromCount(nodeCount, nodeResources)); } private static void assertAcls(List<List<Node>> expected, NodeAcl actual) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index 7e7338ab9ae..2c01cdde932 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -81,7 +81,7 @@ public class AllocationSimulator { var ipConfig = new IP.Config(Set.of("127.0.0.1"), parent.isPresent() ? Set.of() : getAdditionalIP()); return new Node("fake", ipConfig, hostname, parent, flavor, Status.initial(), parent.isPresent() ? Node.State.ready : Node.State.active, allocation(tenant, flavor), History.empty(), - parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty()); + parent.isPresent() ? NodeType.tenant : NodeType.host, new Reports(), Optional.empty(), Optional.empty()); } private Set<String> getAdditionalIP() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index e2015cfd30a..ba9a04573e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -37,15 +37,15 @@ public class DockerHostCapacityTest { @Before public void setup() { - doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + doAnswer(invocation -> invocation.getArguments()[1]).when(hostResourcesCalculator).availableCapacityOf(any(), any()); // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); // Create three docker hosts - host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); - host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), NodeType.host); + host1 = Node.create("host1", new IP.Config(Set.of("::1"), generateIPs(2, 4)), "host1", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host2 = Node.create("host2", new IP.Config(Set.of("::11"), generateIPs(12, 3)), "host2", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); + host3 = Node.create("host3", new IP.Config(Set.of("::21"), generateIPs(22, 1)), "host3", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host); // Add two containers to host1 var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant); @@ -95,9 +95,9 @@ public class DockerHostCapacityTest { capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { - NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[1]; return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); - }).when(hostResourcesCalculator).availableCapacityOf(any()); + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); @@ -110,7 +110,7 @@ public class DockerHostCapacityTest { // Dev host can assign both configserver and tenant containers. var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); - var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost); + var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("devhost"), Optional.empty(), NodeType.devhost); var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java new file mode 100644 index 00000000000..70a57715d13 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class DockerImagesTest { + + @Test + public void image_selection() { + var flagSource = new InMemoryFlagSource(); + var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + + // Host uses tenant default image (for preload purposes) + var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); + var hosts = tester.makeReadyNodes(2, "default", NodeType.host); + tester.deployZoneApp(); + for (var host : hosts) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host)); + } + + // Tenant node uses tenant default image + var resources = new NodeResources(2, 8, 50, 1); + for (var host : hosts) { + var nodes = tester.makeReadyVirtualDockerNodes(2, resources, host.hostname()); + for (var node : nodes) { + assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + + // Allocated containers uses overridden image when feature flag is set + var app = tester.makeApplicationId(); + var nodes = tester.deploy(app, Capacity.fromCount(2, resources)); + var customImage = DockerImage.fromString("docker.example.com/vespa/hosted"); + flagSource.withStringFlag(Flags.DOCKER_IMAGE_OVERRIDE.id(), customImage.asString()); + for (var node : nodes) { + assertEquals(customImage, tester.nodeRepository().dockerImages().dockerImageFor(node)); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index b731ab309a6..1e59add8304 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -107,6 +108,45 @@ public class DockerProvisioningTest { assertEquals(nodeCount, activeNodes.size()); } + @Test + public void reservations_are_respected() { + NodeResources resources = new NodeResources(10, 10, 10, 10); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + TenantName tenant1 = TenantName.from("tenant1"); + TenantName tenant2 = TenantName.from("tenant2"); + ApplicationId application1_1 = ApplicationId.from(tenant1, ApplicationName.from("application1"), InstanceName.defaultName()); + ApplicationId application2_1 = ApplicationId.from(tenant2, ApplicationName.from("application1"), InstanceName.defaultName()); + ApplicationId application2_2 = ApplicationId.from(tenant2, ApplicationName.from("application2"), InstanceName.defaultName()); + + tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1); + tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1); + tester.deployZoneApp(); + + Version wantedVespaVersion = Version.fromString("6.39"); + List<HostSpec> nodes = tester.prepare(application2_1, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 6, 1, resources); + assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1 + tester.activate(application2_1, nodes); + + try { + tester.prepare(application2_2, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 5, 1, resources); + fail("Expected exception"); + } + catch (OutOfCapacityException e) { + // Success: Not enough nonreserved hosts left + } + + nodes = tester.prepare(application1_1, + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + 10, 1, resources); + assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester); + tester.activate(application1_1, nodes); + assertNodeParentReservation(tester.getNodes(application1_1).asList(), Optional.empty(), tester); // Reservation is cleared after activation + } + /** Exclusive app first, then non-exclusive: Should give the same result as below */ @Test public void docker_application_deployment_with_exclusive_app_first() { @@ -260,4 +300,16 @@ public class DockerProvisioningTest { tester.activate(application, hosts); } + private void assertNodeParentReservation(List<Node> nodes, Optional<TenantName> reservation, ProvisioningTester tester) { + for (Node node : nodes) + assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo()); + } + + private void assertHostSpecParentReservation(List<HostSpec> hostSpecs, Optional<TenantName> reservation, ProvisioningTester tester) { + for (HostSpec hostSpec : hostSpecs) { + Node node = tester.nodeRepository().getNode(hostSpec.hostname()).get(); + assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo()); + } + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index d1498507a7c..53fecbf6095 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -423,7 +423,9 @@ public class DynamicDockerAllocationTest { } private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), NodeType.tenant); + Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, + Optional.of(parentHostname), Optional.empty(), new Flavor(flavor), Optional.empty(), NodeType.tenant + ); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.flavor().resources(), Instant.now()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index c0e06ef3dff..0ad7d37d13b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -47,6 +46,7 @@ import static org.junit.Assert.fail; * @author freva */ public class InPlaceResizeProvisionTest { + private static final NodeResources smallResources = new NodeResources(2, 4, 8, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); @@ -55,8 +55,7 @@ public class InPlaceResizeProvisionTest { private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false); private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource() - .withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), true); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder() .flagSource(flagSource) .zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); @@ -158,17 +157,6 @@ public class InPlaceResizeProvisionTest { assertTrue("All initial nodes should still be allocated to the application", initialHostnames.isEmpty()); } - @Test(expected = OutOfCapacityException.class) - public void no_in_place_resize_if_flag_not_set() { - flagSource.withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), false); - addParentHosts(4, mediumResources.with(fast).with(local)); - - new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); - - new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources); - } - @Test(expected = OutOfCapacityException.class) public void cannot_inplace_decrease_resources_while_increasing_cluster_size() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 6dfca4d2c04..ee9a582c4db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.google.common.collect.Iterators; @@ -40,15 +40,14 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); - private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void provision_load_balancer() { - Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); - Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList(); + Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList(); + Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList(); ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); @@ -80,7 +79,7 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); - LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0); assertEquals(2, loadBalancer.instance().reals().size()); assertTrue("Failed node is removed", loadBalancer.instance().reals().stream() .map(Real::hostname) @@ -158,7 +157,7 @@ public class LoadBalancerProvisionerTest { var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), true, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs"))); - Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0); + Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0); assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty()); assignIps(tester.nodeRepository().getNodes(app1)); tester.activate(app1, nodes); @@ -189,7 +188,7 @@ public class LoadBalancerProvisionerTest { clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList()); } @Test @@ -197,12 +196,12 @@ public class LoadBalancerProvisionerTest { tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content, ClusterSpec.Id.from("tenant-host")))); assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty()); - assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList()); + assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList()); } @Test public void provision_load_balancer_combined_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList(); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster)); @@ -247,7 +246,7 @@ public class LoadBalancerProvisionerTest { } private void assignIps(List<Node> nodes) { - try (var lock = tester.nodeRepository().lockAllocation()) { + try (var lock = tester.nodeRepository().lockUnallocated()) { for (int i = 0; i < nodes.size(); i++) { tester.nodeRepository().write(nodes.get(i).with(IP.Config.EMPTY.with(Set.of("127.0.0." + i))), lock); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java index 718abf7d73d..1d9c135b4b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java @@ -124,18 +124,26 @@ public class PrioritizableNodeTest { } private static Node node(String hostname, Node.State state) { - return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(new NodeResources(2, 2, 2, 2)), - Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty()); + return new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(new NodeResources(2, 2, 2, 2)), + Status.initial(), state, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), + Optional.empty(), Optional.empty()); } private static PrioritizableNode node(String hostname, NodeResources nodeResources, NodeResources allocatedHostResources, // allocated before adding nodeResources NodeResources totalHostResources) { - Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), new Flavor(nodeResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, new Reports(), Optional.empty()); - Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), new Flavor(totalHostResources), - Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, new Reports(), Optional.empty()); - return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, false, true, false); + Node node = new Node(hostname, new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.of(hostname + "parent"), + new Flavor(nodeResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.tenant, + new Reports(), Optional.empty(), Optional.empty()); + Node parent = new Node(hostname + "parent", new IP.Config(Set.of("::1"), Set.of()), hostname, Optional.empty(), + new Flavor(totalHostResources), + Status.initial(), Node.State.ready, Optional.empty(), History.empty(), NodeType.host, + new Reports(), Optional.empty(), Optional.empty()); + return new PrioritizableNode(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), + false, false, true, false); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 679c22e4df2..85a6ed31073 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -87,7 +87,7 @@ public class ProvisioningTester { this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver, - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true); + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, flagSource); this.orchestrator = orchestrator; ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); @@ -241,13 +241,16 @@ public class ProvisioningTester { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type) { - return makeReadyNodes(n, asFlavor(flavor, type), type, 0); + return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, 0); } public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type) { - return makeReadyNodes(n, new Flavor(resources), type, 0); + return makeReadyNodes(n, new Flavor(resources), Optional.empty(), type, 0); } public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, new Flavor(resources), type, ipAddressPoolSize); + return makeReadyNodes(n, resources, Optional.empty(), type, ipAddressPoolSize); + } + public List<Node> makeReadyNodes(int n, NodeResources resources, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) { + return makeReadyNodes(n, new Flavor(resources), reservedTo, type, ipAddressPoolSize); } public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) { @@ -255,9 +258,9 @@ public class ProvisioningTester { } public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { - return makeProvisionedNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack); + return makeProvisionedNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize, dualStack); } - public List<Node> makeProvisionedNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { + public List<Node> makeProvisionedNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -299,6 +302,7 @@ public class ProvisioningTester { new IP.Config(hostIps, ipAddressPool), Optional.empty(), flavor, + reservedTo, type)); } nodes = nodeRepository.addNodes(nodes); @@ -315,11 +319,12 @@ public class ProvisioningTester { nameResolver.addRecord(hostname, ipv4); Node node = nodeRepository.createNode(hostname, - hostname, - new IP.Config(Set.of(ipv4), Set.of()), - Optional.empty(), - nodeFlavors.getFlavorOrThrow(flavor), - NodeType.config); + hostname, + new IP.Config(Set.of(ipv4), Set.of()), + Optional.empty(), + nodeFlavors.getFlavorOrThrow(flavor), + Optional.empty(), + NodeType.config); nodes.add(node); } @@ -338,17 +343,20 @@ public class ProvisioningTester { } public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize); + return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize); } - public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize) { - return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false); + public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) { + return makeReadyNodes(n, flavor, reservedTo, type, ipAddressPoolSize, false); } public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack); } public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) { - List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack); + return makeReadyNodes(n, flavor, Optional.empty(), type, ipAddressPoolSize, dualStack); + } + public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { + List<Node> nodes = makeProvisionedNodes(n, flavor, reservedTo, type, ipAddressPoolSize, dualStack); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); } @@ -410,21 +418,20 @@ public class ProvisioningTester { activate(applicationId, Set.copyOf(list)); } + public List<Node> deploy(ApplicationId application, Capacity capacity) { + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), + Version.fromString("6.42"), false); + List<HostSpec> prepared = prepare(application, cluster, capacity, 1); + activate(application, Set.copyOf(prepared)); + return getNodes(application, Node.State.active).asList(); + } + + /** Returns the hosts from the input list which are not retired */ public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) { - long actualNodesWithFlavor = hostSpecs.stream() - .map(HostSpec::hostname) - .map(this::getNodeFlavor) - .map(Flavor::name) - .filter(name -> name.equals(flavor)) - .count(); - assertEquals(expectedCount, actualNodesWithFlavor); - } - public void assertAllocatedOn(String explanation, String hostFlavor, ApplicationId app) { for (Node node : nodeRepository.getNodes(app)) { Node parent = nodeRepository.getNode(node.parentHostname().get()).get(); @@ -432,17 +439,6 @@ public class ProvisioningTester { } } - public void printFreeResources() { - for (Node host : nodeRepository().getNodes(NodeType.host)) { - NodeResources free = host.flavor().resources(); - for (Node child : nodeRepository().getNodes(NodeType.tenant)) { - if (child.parentHostname().get().equals(host.hostname())) - free = free.subtract(child.flavor().resources()); - } - System.out.println(host.flavor().name() + " node. Free resources: " + free); - } - } - public int hostFlavorCount(String hostFlavor, ApplicationId app) { return (int)nodeRepository().getNodes(app).stream() .map(n -> nodeRepository().getNode(n.parentHostname().get()).get()) @@ -450,10 +446,6 @@ public class ProvisioningTester { .count(); } - private Flavor getNodeFlavor(String hostname) { - return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname)); - } - public static final class Builder { private Curator curator; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 18f0d989900..c26614c630c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -6,12 +6,15 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.text.Utf8; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.After; import org.junit.Before; import org.junit.ComparisonFailure; @@ -23,6 +26,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -106,7 +110,7 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asHostJson("parent2.yahoo.com", "large-variant", "127.0.127.1", "::127:1") + "," + + asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -214,9 +218,6 @@ public class RestApiTest { Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -225,6 +226,9 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), @@ -232,6 +236,9 @@ public class RestApiTest { assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); + ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName())) + .suspend(new HostName("host4.yahoo.com")); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); } @@ -292,7 +299,7 @@ public class RestApiTest { // Attempt to POST host node with already assigned IP assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", "127.0.2.1") + "]", + "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); @@ -304,7 +311,7 @@ public class RestApiTest { // Node types running a single container can share their IP address with child node assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -320,7 +327,7 @@ public class RestApiTest { // ... nor with child node on different host assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -534,11 +541,13 @@ public class RestApiTest { " \"actualCpuCores\": {" + " \"createdMillis\": 1, " + " \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," + + " \"type\": \"HARD_FAIL\"," + " \"value\":2" + " }," + " \"diskSpace\": {" + " \"createdMillis\": 2, " + " \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," + + " \"type\": \"HARD_FAIL\"," + " \"details\": {" + " \"inGib\": 3," + " \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" + @@ -921,14 +930,15 @@ public class RestApiTest { "\"flavor\":\"" + flavor + "\"}"; } - private static String asHostJson(String hostname, String flavor, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, ipAddress); + private static String asHostJson(String hostname, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { + return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + + (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + ", \"type\":\"" + nodeType + "\"}"; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index f663f4adb8d..af3552945d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -27,14 +27,15 @@ "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", "wantedVespaVersion": "6.42.0", "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, + "allowedToBeDown": true, + "suspendedSinceMillis": 0, "rebootGeneration": 3, "currentRebootGeneration": 1, "vespaVersion": "6.43.0", "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0", "failCount": 1, "wantToRetire": true, - "wantToDeprovision": true, + "wantToDeprovision": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json index d33c1c9e743..a3d53798d7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -65,6 +65,7 @@ "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json index 4119e46e225..67b8d67c7f1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -62,11 +62,13 @@ "actualCpuCores": { "createdMillis": 1, "description": "Actual number of CPU cores (2) differs from spec (4)", + "type":"HARD_FAIL", "value": 2 }, "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index 8be034cb036..ecc497172c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -6,6 +6,7 @@ "hostname": "parent2.yahoo.com", "openStackId": "parent2.yahoo.com", "flavor": "large-variant", + "reservedTo": "myTenant", "cpuCores": 64.0, "resources": {"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, "environment": "BARE_METAL", diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java index 3f14579dfba..2a582020bb3 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java @@ -18,6 +18,7 @@ public class GetHostResponse { public static final String FIELD_NAME_HOSTNAME = "hostname"; public static final String FIELD_NAME_STATE = "state"; + public static final String FIELD_NAME_SUSPENDED_SINCE = "suspendedSince"; public static final String FIELD_NAME_APPLICATION_URL = "applicationUrl"; public static final String FIELD_NAME_SERVICES = "services"; @@ -25,15 +26,18 @@ public class GetHostResponse { private final String state; private final String applicationUrl; private final List<HostService> services; + private final String suspendedSince; @JsonCreator public GetHostResponse( @JsonProperty(FIELD_NAME_HOSTNAME) String hostname, @JsonProperty(FIELD_NAME_STATE) String state, + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) String suspendedSince, @JsonProperty(FIELD_NAME_APPLICATION_URL) String applicationUrl, @JsonProperty(FIELD_NAME_SERVICES) List<HostService> services) { this.hostname = hostname; this.state = state; + this.suspendedSince = suspendedSince; this.applicationUrl = applicationUrl; this.services = services; } @@ -48,6 +52,11 @@ public class GetHostResponse { return state; } + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) + public String suspendedSince() { + return suspendedSince; + } + @JsonProperty(FIELD_NAME_APPLICATION_URL) public String applicationUrl() { return applicationUrl; @@ -65,12 +74,13 @@ public class GetHostResponse { GetHostResponse that = (GetHostResponse) o; return Objects.equals(hostname, that.hostname) && Objects.equals(state, that.state) && + Objects.equals(suspendedSince, that.suspendedSince) && Objects.equals(applicationUrl, that.applicationUrl) && Objects.equals(services, that.services); } @Override public int hashCode() { - return Objects.hash(hostname, state, applicationUrl, services); + return Objects.hash(hostname, state, suspendedSince, applicationUrl, services); } } diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java new file mode 100644 index 00000000000..39c93291bad --- /dev/null +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.orchestrator.restapi.wire; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.Objects; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireHostInfo { + private static final String HOST_STATUS_FIELD = "hostStatus"; + private static final String SUSPENDED_SINCE_FIELD = "suspendedSince"; + + private final String hostStatus; + private final String suspendedSinceUtcOrNull; + + /** + * @param hostStatus The host status, e.g. NO_REMARKS. + * @param suspendedSinceUtcOrNull The time the host was suspended in the format + * {@link Instant#toString()}, or null if not suspended + * (NO_REMARKS). + */ + @JsonCreator + public WireHostInfo(@JsonProperty(HOST_STATUS_FIELD) String hostStatus, + @JsonProperty(SUSPENDED_SINCE_FIELD) String suspendedSinceUtcOrNull) { + this.hostStatus = Objects.requireNonNull(hostStatus); + this.suspendedSinceUtcOrNull = suspendedSinceUtcOrNull; + } + + @JsonProperty(HOST_STATUS_FIELD) + public String hostStatus() { return hostStatus; } + + @JsonProperty(SUSPENDED_SINCE_FIELD) + public String getSuspendedSinceUtcOrNull() { return suspendedSinceUtcOrNull; } +} diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 3ebc87010fb..5dd4e7ea87d 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -42,6 +42,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>flags</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -59,6 +65,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java index bda9505d72b..8f5f00af7a0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java @@ -4,23 +4,23 @@ package com.yahoo.vespa.orchestrator; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.util.List; public class Host { private final HostName hostName; - private final HostStatus hostStatus; + private final HostInfo hostInfo; private final ApplicationInstanceReference applicationInstanceReference; private final List<ServiceInstance> serviceInstances; public Host(HostName hostName, - HostStatus hostStatus, + HostInfo hostInfo, ApplicationInstanceReference applicationInstanceReference, List<ServiceInstance> serviceInstances) { this.hostName = hostName; - this.hostStatus = hostStatus; + this.hostInfo = hostInfo; this.applicationInstanceReference = applicationInstanceReference; this.serviceInstances = serviceInstances; } @@ -29,8 +29,8 @@ public class Host { return hostName; } - public HostStatus getHostStatus() { - return hostStatus; + public HostInfo getHostInfo() { + return hostInfo; } public ApplicationInstanceReference getApplicationInstanceReference() { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index 59b320cf501..10fa10f1150 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.List; @@ -49,12 +50,15 @@ public interface Orchestrator { HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException; /** - * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this. + * Returns a lambda, which when invoked with a hostname, returns its current host info. + * + * <p>When invoked multiple times, the hostname/host info mapping is not necessarily consistent. + * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, + * and when doing bulk reads.</p> * - * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads. * @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}. */ - Function<HostName, Optional<HostStatus>> getNodeStatuses(); + Function<HostName, Optional<HostInfo>> getHostResolver(); void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index d3bdaa6dc64..eb6a4119f8a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -1,13 +1,17 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; +import com.yahoo.log.LogLevel; import com.yahoo.time.TimeBudget; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.Optional; +import java.util.logging.Logger; /** * Context for an operation (or suboperation) of the Orchestrator that needs to pass through to the backend, @@ -15,29 +19,52 @@ import java.util.Optional; * * @author hakonhall */ -public class OrchestratorContext { +public class OrchestratorContext implements AutoCloseable { + private static final Logger logger = Logger.getLogger(OrchestratorContext.class.getName()); private static final Duration DEFAULT_TIMEOUT_FOR_SINGLE_OP = Duration.ofSeconds(10); private static final Duration DEFAULT_TIMEOUT_FOR_BATCH_OP = Duration.ofSeconds(60); private static final Duration TIMEOUT_OVERHEAD = Duration.ofMillis(500); + private final Optional<OrchestratorContext> parent; private final Clock clock; private final TimeBudget timeBudget; private final boolean probe; + private final boolean largeLocks; + private final boolean usePermanentlyDownStatus; + + // The key set is the set of applications locked by this context tree: Only the + // root context has a non-empty set. The value is an unlock callback to be called + // when the root context is closed. + private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>(); /** Create an OrchestratorContext for operations on multiple applications. */ - public static OrchestratorContext createContextForMultiAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), false); + public static OrchestratorContext createContextForMultiAppOp(Clock clock, boolean largeLocks) { + return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), + false, largeLocks, false); } /** Create an OrchestratorContext for an operation on a single application. */ public static OrchestratorContext createContextForSingleAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), false); + return createContextForSingleAppOp(clock, false); + } + + public static OrchestratorContext createContextForSingleAppOp(Clock clock, boolean usePermanentlyDownStatus) { + return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), + false, false, usePermanentlyDownStatus); } - private OrchestratorContext(Clock clock, TimeBudget timeBudget, boolean probe) { + private OrchestratorContext(OrchestratorContext parentOrNull, + Clock clock, + TimeBudget timeBudget, + boolean probe, + boolean largeLocks, + boolean usePermanentlyDownStatus) { + this.parent = Optional.ofNullable(parentOrNull); this.clock = clock; this.timeBudget = timeBudget; this.probe = probe; + this.largeLocks = largeLocks; + this.usePermanentlyDownStatus = usePermanentlyDownStatus; } public Duration getTimeLeft() { @@ -53,12 +80,50 @@ public class OrchestratorContext { return probe; } + /** Whether application locks acquired during probing of a batch suspend should be closed after the non-probe is done. */ + public boolean largeLocks() { return largeLocks; } + + /** Whether the PERMANENTLY_DOWN host status should be used (where appropriate). */ + public boolean usePermanentlyDownStatus() { return usePermanentlyDownStatus; } + + /** + * Returns true if 1. large locks is enabled, and 2. + * {@link #registerLockAcquisition(ApplicationInstanceReference, Runnable) registerLockAcquisition} + * has been invoked on any context below the root context that returned true. + */ + public boolean hasLock(ApplicationInstanceReference application) { + return parent.map(p -> p.hasLock(application)).orElseGet(() -> locks.containsKey(application)); + } + + /** + * Returns true if large locks is enabled in the root context, and in case the unlock callback + * will be invoked when the root context is closed. + */ + public boolean registerLockAcquisition(ApplicationInstanceReference application, Runnable unlock) { + if (parent.isPresent()) { + return parent.get().registerLockAcquisition(application, unlock); + } + + if (!largeLocks) { + return false; + } + + if (locks.containsKey(application)) { + unlock.run(); + throw new IllegalStateException("Application " + application + " was already associated with a lock"); + } + + locks.put(application, unlock); + + return true; + } + /** Create OrchestratorContext to use within an application lock. */ public OrchestratorContext createSubcontextWithinLock() { // Move deadline towards past by a fixed amount to ensure there's time to process exceptions and // access ZooKeeper before the lock times out. TimeBudget subTimeBudget = timeBudget.withDeadline(timeBudget.deadline().get().minus(TIMEOUT_OVERHEAD)); - return new OrchestratorContext(clock, subTimeBudget, probe); + return new OrchestratorContext(this, clock, subTimeBudget, probe, largeLocks, usePermanentlyDownStatus); } /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */ @@ -70,9 +135,18 @@ public class OrchestratorContext { deadline = maxDeadline; } - return new OrchestratorContext( - clock, - TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline))), - probe); + TimeBudget timeBudget = TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline))); + return new OrchestratorContext(this, clock, timeBudget, probe, largeLocks, usePermanentlyDownStatus); + } + + @Override + public void close() { + locks.forEach((application, unlock) -> { + try { + unlock.run(); + } catch (RuntimeException e) { + logger.log(LogLevel.ERROR, "Failed run on close : " + e.getMessage()); + } + }); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 0cf654b23b5..fbe6864274c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -12,6 +12,10 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; @@ -27,6 +31,8 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.policy.Policy; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -58,13 +64,16 @@ public class OrchestratorImpl implements Orchestrator { private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; private final ApplicationApiFactory applicationApiFactory; + private final BooleanFlag enableLargeOrchestratorLocks; + private final BooleanFlag retireWithPermanentlyDownFlag; @Inject public OrchestratorImpl(ClusterControllerClientFactory clusterControllerClientFactory, StatusService statusService, OrchestratorConfig orchestratorConfig, InstanceLookupService instanceLookupService, - ConfigserverConfig configServerConfig) + ConfigserverConfig configServerConfig, + FlagSource flagSource) { this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, new ApplicationApiFactory(configServerConfig.zookeeperserver().size())), clusterControllerClientFactory, @@ -72,7 +81,8 @@ public class OrchestratorImpl implements Orchestrator { instanceLookupService, orchestratorConfig.serviceMonitorConvergenceLatencySeconds(), Clock.systemUTC(), - new ApplicationApiFactory(configServerConfig.zookeeperserver().size())); + new ApplicationApiFactory(configServerConfig.zookeeperserver().size()), + flagSource); } public OrchestratorImpl(Policy policy, @@ -81,7 +91,8 @@ public class OrchestratorImpl implements Orchestrator { InstanceLookupService instanceLookupService, int serviceMonitorConvergenceLatencySeconds, Clock clock, - ApplicationApiFactory applicationApiFactory) + ApplicationApiFactory applicationApiFactory, + FlagSource flagSource) { this.policy = policy; this.clusterControllerClientFactory = clusterControllerClientFactory; @@ -90,6 +101,8 @@ public class OrchestratorImpl implements Orchestrator { this.instanceLookupService = instanceLookupService; this.clock = clock; this.applicationApiFactory = applicationApiFactory; + this.enableLargeOrchestratorLocks = Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.bindTo(flagSource); + this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource); } @Override @@ -101,9 +114,10 @@ public class OrchestratorImpl implements Orchestrator { .filter(serviceInstance -> hostName.equals(serviceInstance.hostName())) .collect(Collectors.toList()); + HostInfo hostInfo = statusService.getHostInfo(applicationInstance.reference(), hostName); HostStatus hostStatus = getNodeStatus(applicationInstance.reference(), hostName); - return new Host(hostName, hostStatus, applicationInstance.reference(), serviceInstances); + return new Host(hostName, hostInfo, applicationInstance.reference(), serviceInstances); } @Override @@ -112,11 +126,9 @@ public class OrchestratorImpl implements Orchestrator { } @Override - public Function<HostName, Optional<HostStatus>> getNodeStatuses() { - Function<ApplicationInstanceReference, Set<HostName>> suspendedHosts = statusService.getSuspendedHostsByApplication(); + public Function<HostName, Optional<HostInfo>> getHostResolver() { return hostName -> instanceLookupService.findInstanceByHost(hostName) - .map(application -> suspendedHosts.apply(application.reference()).contains(hostName) - ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS); + .map(application -> statusService.getHostInfo(application.reference(), hostName)); } @Override @@ -141,6 +153,12 @@ public class OrchestratorImpl implements Orchestrator { * monitoring will have had time to catch up. Since we don't want do the delay with the lock held, * and the host status service's locking functionality does not support something like condition * variables or Object.wait(), we break out here, releasing the lock before delaying. + * + * 2020-02-07: We should utilize suspendedSince timestamp on the HostInfo: The above + * is equivalent to guaranteeing a minimum time after suspendedSince, before checking + * the health with service monitor. This should for all practical purposes remove + * the amount of time in this sleep. + * Caveat: Cannot be implemented before lingering HostInfo has been fixed (VESPA-17546). */ sleep(serviceMonitorConvergenceLatencySeconds, TimeUnit.SECONDS); @@ -149,16 +167,22 @@ public class OrchestratorImpl implements Orchestrator { OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); try (MutableStatusRegistry statusRegistry = statusService .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { - HostStatus currentHostState = statusRegistry.getHostStatus(hostName); - - if (HostStatus.NO_REMARKS == currentHostState) { + HostStatus currentHostState = statusRegistry.getHostInfo(hostName).status(); + if (currentHostState == HostStatus.NO_REMARKS) { return; } - ApplicationInstanceStatus appStatus = statusRegistry.getStatus(); - if (appStatus == ApplicationInstanceStatus.NO_REMARKS) { - policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); + // In 2 cases the resume will appear to succeed (no exception thrown), + // but the host status and content cluster states will not be changed accordingly: + // 1. When host is permanently down: the host will be removed from the application asap. + // 2. The whole application is down: the content cluster states are set to maintenance, + // and the host may be taken down manually at any moment. + if (currentHostState == HostStatus.PERMANENTLY_DOWN || + statusRegistry.getStatus() == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { + return; } + + policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); } } @@ -174,7 +198,10 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstance appInstance = getApplicationInstance(hostName); NodeGroup nodeGroup = new NodeGroup(appInstance, hostName); - OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); + boolean usePermanentlyDownStatus = retireWithPermanentlyDownFlag + .with(FetchVector.Dimension.HOSTNAME, hostName.s()) + .value(); + OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock, usePermanentlyDownStatus); try (MutableStatusRegistry statusRegistry = statusService .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, statusRegistry, @@ -231,17 +258,20 @@ public class OrchestratorImpl implements Orchestrator { @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { - OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock); + boolean largeLocks = enableLargeOrchestratorLocks + .with(FetchVector.Dimension.HOSTNAME, parentHostname.s()) + .value(); + try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock, largeLocks)) { + List<NodeGroup> nodeGroupsOrderedByApplication; + try { + nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); + } catch (HostNameNotFoundException e) { + throw new BatchHostNameNotFoundException(parentHostname, hostNames, e); + } - List<NodeGroup> nodeGroupsOrderedByApplication; - try { - nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); - } catch (HostNameNotFoundException e) { - throw new BatchHostNameNotFoundException(parentHostname, hostNames, e); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false); } - - suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true); - suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false); } private void suspendAllNodeGroups(OrchestratorContext context, @@ -254,6 +284,8 @@ public class OrchestratorImpl implements Orchestrator { suspendGroup(context.createSubcontextForSingleAppOp(probe), nodeGroup); } catch (HostStateChangeDeniedException e) { throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e); + } catch (UncheckedTimeoutException e) { + throw e; } catch (RuntimeException e) { throw new BatchInternalErrorException(parentHostname, nodeGroup, e); } @@ -318,7 +350,7 @@ public class OrchestratorImpl implements Orchestrator { } private HostStatus getNodeStatus(ApplicationInstanceReference applicationRef, HostName hostName) { - return statusService.getHostStatus(applicationRef, hostName); + return statusService.getHostInfo(applicationRef, hostName).status(); } private void setApplicationStatus(ApplicationId appId, ApplicationInstanceStatus status) @@ -335,9 +367,15 @@ public class OrchestratorImpl implements Orchestrator { if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { ApplicationInstance application = getApplicationInstance(appRef); + HostInfos hostInfosSnapshot = statusRegistry.getHostInfos(); + // Mark it allowed to be down before we manipulate the clustercontroller OrchestratorUtil.getHostsUsedByApplicationInstance(application) - .forEach(h -> statusRegistry.setHostState(h, HostStatus.ALLOWED_TO_BE_DOWN)); + .stream() + // This filter also ensures host status is not modified if a suspended host + // has status != ALLOWED_TO_BE_DOWN. + .filter(hostname -> !hostInfosSnapshot.getOrNoRemarks(hostname).status().isSuspended()) + .forEach(hostname -> statusRegistry.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN)); // If the clustercontroller throws an error the nodes will be marked as allowed to be down // and be set back up on next resume invocation. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java index 2e85713d323..a6353e39610 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.List; +import java.util.function.Predicate; /** * The API a Policy has access to @@ -29,10 +30,12 @@ public interface ApplicationApi { ApplicationInstanceStatus getApplicationStatus(); void setHostState(OrchestratorContext context, HostName hostName, HostStatus status); - List<HostName> getNodesInGroupWithStatus(HostStatus status); + List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate); + default List<HostName> getNodesInGroupWithStatus(HostStatus requiredStatus) { + return getNodesInGroupWith(status -> status == requiredStatus); + } List<StorageNode> getStorageNodesInGroupInClusterOrder(); List<StorageNode> getUpStorageNodesInGroupInClusterOrder(); - List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder(); - + List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index eb5dcf790ba..cf6946fa7f8 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; @@ -17,10 +18,9 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -34,7 +34,7 @@ public class ApplicationApiImpl implements ApplicationApi { private final NodeGroup nodeGroup; private final MutableStatusRegistry hostStatusService; private final List<ClusterApi> clusterInOrder; - private final Map<HostName, HostStatus> hostStatusMap; + private final HostInfos hostInfos; public ApplicationApiImpl(NodeGroup nodeGroup, MutableStatusRegistry hostStatusService, @@ -44,10 +44,8 @@ public class ApplicationApiImpl implements ApplicationApi { this.nodeGroup = nodeGroup; this.hostStatusService = hostStatusService; Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance); - this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(), - hostName -> hostStatusService.getSuspendedHosts().contains(hostName) - ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS)); - this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory, numberOfConfigServers); + this.hostInfos = hostStatusService.getHostInfos(); + this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers); } @Override @@ -56,7 +54,7 @@ public class ApplicationApiImpl implements ApplicationApi { } private HostStatus getHostStatus(HostName hostName) { - return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS); + return hostInfos.getOrNoRemarks(hostName).status(); } @Override @@ -65,9 +63,9 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override - public List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() { + public List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder() { return getStorageNodesInGroupInClusterOrder().stream() - .filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) + .filter(storageNode -> getHostStatus(storageNode.hostName()).isSuspended()) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); } @@ -106,13 +104,14 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override - public List<HostName> getNodesInGroupWithStatus(HostStatus status) { + public List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate) { return nodeGroup.getHostNames().stream() - .filter(hostName -> getHostStatus(hostName) == status) + .filter(hostName -> statusPredicate.test(getHostStatus(hostName))) .collect(Collectors.toList()); } - private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, Map<HostName, HostStatus> hostStatusMap, + private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, + HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, int numberOfConfigServers) { Set<ServiceCluster> clustersInGroup = getServiceClustersInGroup(nodeGroup); @@ -121,7 +120,7 @@ public class ApplicationApiImpl implements ApplicationApi { this, serviceCluster, nodeGroup, - hostStatusMap, + hostInfos, clusterControllerClientFactory, numberOfConfigServers)) .sorted(ApplicationApiImpl::compareClusters) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 2aaebb18aa6..b747d8c2e22 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.Collections; @@ -27,7 +28,7 @@ class ClusterApiImpl implements ClusterApi { private final ApplicationApi applicationApi; private final ServiceCluster serviceCluster; private final NodeGroup nodeGroup; - private final Map<HostName, HostStatus> hostStatusMap; + private final HostInfos hostInfos; private final ClusterControllerClientFactory clusterControllerClientFactory; private final Set<ServiceInstance> servicesInGroup; private final Set<ServiceInstance> servicesDownInGroup; @@ -50,13 +51,13 @@ class ClusterApiImpl implements ClusterApi { public ClusterApiImpl(ApplicationApi applicationApi, ServiceCluster serviceCluster, NodeGroup nodeGroup, - Map<HostName, HostStatus> hostStatusMap, + HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, int numberOfConfigServers) { this.applicationApi = applicationApi; this.serviceCluster = serviceCluster; this.nodeGroup = nodeGroup; - this.hostStatusMap = hostStatusMap; + this.hostInfos = hostInfos; this.clusterControllerClientFactory = clusterControllerClientFactory; Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality = @@ -144,7 +145,7 @@ class ClusterApiImpl implements ClusterApi { public String nodesAllowedToBeDownNotInGroupDescription() { return servicesNotInGroup.stream() .map(ServiceInstance::hostName) - .filter(hostName -> hostStatus(hostName) == HostStatus.ALLOWED_TO_BE_DOWN) + .filter(hostName -> hostStatus(hostName).isSuspended()) .sorted() .distinct() .collect(Collectors.toList()) @@ -206,11 +207,11 @@ class ClusterApiImpl implements ClusterApi { } private HostStatus hostStatus(HostName hostName) { - return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS); + return hostInfos.getOrNoRemarks(hostName).status(); } private boolean serviceEffectivelyDown(ServiceInstance service) { - if (hostStatus(service.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) { + if (hostStatus(service.hostName()).isSuspended()) { return true; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index a9b9736ebfb..f6a1e4f91f0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -30,7 +30,9 @@ public class HostedVespaPolicy implements Policy { private final ClusterControllerClientFactory clusterControllerClientFactory; private final ApplicationApiFactory applicationApiFactory; - public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory, ApplicationApiFactory applicationApiFactory) { + public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, + ClusterControllerClientFactory clusterControllerClientFactory, + ApplicationApiFactory applicationApiFactory) { this.clusterPolicy = clusterPolicy; this.clusterControllerClientFactory = clusterControllerClientFactory; this.applicationApiFactory = applicationApiFactory; @@ -60,10 +62,11 @@ public class HostedVespaPolicy implements Policy { public void releaseSuspensionGrant(OrchestratorContext context, ApplicationApi application) throws HostStateChangeDeniedException { // Always defer to Cluster Controller whether it's OK to resume storage node - for (StorageNode storageNode : application.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder()) { + for (StorageNode storageNode : application.getSuspendedStorageNodesInGroupInReverseClusterOrder()) { storageNode.setNodeState(context, ClusterControllerNodeState.UP); } + // In particular, we're not modifying the state of PERMANENTLY_DOWN nodes. for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.ALLOWED_TO_BE_DOWN)) { application.setHostState(context, hostName, HostStatus.NO_REMARKS); } @@ -92,9 +95,15 @@ public class HostedVespaPolicy implements Policy { storageNode.setNodeState(context, ClusterControllerNodeState.DOWN); } - // Ensure all nodes in the group are marked as allowed to be down - for (HostName hostName : applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { - applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); + if (context.usePermanentlyDownStatus()) { + // Ensure all nodes in the group are marked as permanently down + for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.PERMANENTLY_DOWN)) { + applicationApi.setHostState(context, hostName, HostStatus.PERMANENTLY_DOWN); + } + } else { + for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.ALLOWED_TO_BE_DOWN)) { + applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); + } } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 4bb93ffa3cb..fc5c5eb5004 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.net.URI; +import java.time.Instant; import java.util.List; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -72,7 +73,8 @@ public class HostResource implements HostApi { return new GetHostResponse( host.getHostName().s(), - host.getHostStatus().name(), + host.getHostInfo().status().name(), + host.getHostInfo().suspendedSince().map(Instant::toString).orElse(null), applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index bd265ed39e4..fbb8f445db0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -14,7 +14,9 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.service.manager.MonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; @@ -29,9 +31,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -81,15 +84,24 @@ public class InstanceResource { = instanceLookupService.findInstanceById(instanceId) .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build())); - Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference()); - Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance) - .stream() - .collect(Collectors.toMap(hostName -> hostName, - hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name() - : HostStatus.NO_REMARKS.name())); + HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference()); + TreeMap<HostName, WireHostInfo> hostStatusMap = + getHostsUsedByApplicationInstance(applicationInstance) + .stream() + .collect(Collectors.toMap( + hostName -> hostName, + hostName -> hostInfoToWire(hostInfos.getOrNoRemarks(hostName)), + (u, v) -> { throw new IllegalStateException(); }, + TreeMap::new)); return InstanceStatusResponse.create(applicationInstance, hostStatusMap); } + private WireHostInfo hostInfoToWire(HostInfo hostInfo) { + String hostStatusString = hostInfo.status().asString(); + String suspendedSinceUtcOrNull = hostInfo.suspendedSince().map(Instant::toString).orElse(null); + return new WireHostInfo(hostStatusString, suspendedSinceUtcOrNull); + } + @GET @Path("/{instanceId}/slobrok") @Produces(MediaType.APPLICATION_JSON) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java index c2ea0c9eddf..313c73e5c68 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java @@ -4,9 +4,12 @@ package com.yahoo.vespa.orchestrator.resources; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.Collectors; /* * @author andreer @@ -14,16 +17,16 @@ import java.util.Objects; public class InstanceStatusResponse { private final ApplicationInstance applicationInstance; - private final Map<HostName, String> hostStates; + private final TreeMap<HostName, WireHostInfo> hostInfos; - private InstanceStatusResponse(ApplicationInstance applicationInstance, Map<HostName, String> hostStates) { + private InstanceStatusResponse(ApplicationInstance applicationInstance, TreeMap<HostName, WireHostInfo> hostInfos) { this.applicationInstance = applicationInstance; - this.hostStates = hostStates; + this.hostInfos = hostInfos; } public static InstanceStatusResponse create( ApplicationInstance applicationInstance, - Map<HostName, String> hostStates) { + TreeMap<HostName, WireHostInfo> hostStates) { return new InstanceStatusResponse(applicationInstance, hostStates); } @@ -34,14 +37,24 @@ public class InstanceStatusResponse { @JsonProperty("hostStates") public Map<HostName, String> hostStates() { - return hostStates; + // TODO: Remove this once all clients have been moved to hostStatus. + return hostInfos.entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> entry.getValue().hostStatus() + )); + } + + @JsonProperty("hostInfos") + public TreeMap<HostName, WireHostInfo> hostInfos() { + return hostInfos; } @Override public String toString() { return "InstanceStatusResponse{" + "applicationInstance=" + applicationInstance + - ", hostStates=" + hostStates + + ", hostInfos=" + hostInfos + '}'; } @@ -51,11 +64,11 @@ public class InstanceStatusResponse { if (o == null || getClass() != o.getClass()) return false; InstanceStatusResponse that = (InstanceStatusResponse) o; return Objects.equals(applicationInstance, that.applicationInstance) && - Objects.equals(hostStates, that.hostStates); + Objects.equals(hostInfos, that.hostInfos); } @Override public int hashCode() { - return Objects.hash(applicationInstance, hostStates); + return Objects.hash(applicationInstance, hostInfos); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfo.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfo.java new file mode 100644 index 00000000000..3dd597f2436 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfo.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * ZooKeeper-backed information Host status information about a host. + * + * @author hakonhall + */ +// @Immutable +public class HostInfo { + private final HostStatus status; + private final Optional<Instant> suspendedSince; + + public static HostInfo createSuspended(HostStatus status, Instant suspendedSince) { + if (!status.isSuspended()) { + throw new IllegalArgumentException(status + " is not a suspended-status"); + } + + return new HostInfo(status, Optional.of(suspendedSince)); + } + + public static HostInfo createNoRemarks() { + return new HostInfo(HostStatus.NO_REMARKS, Optional.empty()); + } + + private HostInfo(HostStatus status, Optional<Instant> suspendedSince) { + this.status = Objects.requireNonNull(status); + this.suspendedSince = Objects.requireNonNull(suspendedSince); + } + + public HostStatus status() { return status; } + + /** + * The instant the host status was set to a suspended status. Is preserved when transitioning + * between suspended statuses. Returns empty if and only if NO_REMARKS. + */ + public Optional<Instant> suspendedSince() { return suspendedSince; } + + @Override + public String toString() { + return "HostInfo{" + + ", status=" + status + + ", suspendedSince=" + suspendedSince + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HostInfo hostInfo = (HostInfo) o; + return status == hostInfo.status && + suspendedSince.equals(hostInfo.suspendedSince); + } + + @Override + public int hashCode() { + return Objects.hash(status, suspendedSince); + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java new file mode 100644 index 00000000000..1acd82662a4 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java @@ -0,0 +1,31 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.vespa.applicationmodel.HostName; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Collection of the suspended hosts of an application. + * + * @author hakonhall + */ +// @Immutable +public class HostInfos { + private final Map<HostName, HostInfo> hostInfos; + + public HostInfos(Map<HostName, HostInfo> hostInfos) { + this.hostInfos = Map.copyOf(hostInfos); + } + + public HostInfos() { + this.hostInfos = Map.of(); + } + + /** Get host info for hostname, returning a NO_REMARKS HostInfo if unknown. */ + public HostInfo getOrNoRemarks(HostName hostname) { + return hostInfos.getOrDefault(hostname, HostInfo.createNoRemarks()); + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java new file mode 100644 index 00000000000..680f3cbcc6d --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorCounter; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author hakonhall + */ +public class HostInfosCache implements HostInfosService { + final static String HOST_STATUS_CACHE_COUNTER_PATH = "/vespa/host-status-service-cache-counter"; + + private final CuratorCounter counter; + private final HostInfosService wrappedService; + + private final AtomicLong cacheGeneration; + private final Map<ApplicationInstanceReference, HostInfos> suspendedHosts = new ConcurrentHashMap<>(); + + HostInfosCache(Curator curator, HostInfosService wrappedService) { + this.counter = new CuratorCounter(curator, HOST_STATUS_CACHE_COUNTER_PATH); + this.wrappedService = wrappedService; + this.cacheGeneration = new AtomicLong(counter.get()); + } + + public void refreshCache() { + long newCacheGeneration = counter.get(); + if (cacheGeneration.getAndSet(newCacheGeneration) != newCacheGeneration) { + suspendedHosts.clear(); + } + } + + public HostInfos getCachedHostInfos(ApplicationInstanceReference application) { + return suspendedHosts.computeIfAbsent(application, wrappedService::getHostInfos); + } + + @Override + public HostInfos getHostInfos(ApplicationInstanceReference application) { + refreshCache(); + return getCachedHostInfos(application); + } + + @Override + public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) { + boolean isException = true; + boolean modified = false; + try { + modified = wrappedService.setHostStatus(application, hostName, hostStatus); + isException = false; + } finally { + if (modified || isException) { + // ensure the next get, on any config server, will repopulate the cache from zk. + counter.next(); + } + } + + return modified; + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java new file mode 100644 index 00000000000..f5c079f9ba3 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java @@ -0,0 +1,15 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; + +/** + * @author hakonhall + */ +interface HostInfosService { + HostInfos getHostInfos(ApplicationInstanceReference application); + + /** Returns false if it is known that the operation was a no-op. */ + boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus); +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java index d00de04bf63..330e53f9586 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java @@ -2,11 +2,26 @@ package com.yahoo.vespa.orchestrator.status; /** - * Enumeration of the different status' a host can have. + * Enumeration of the different statuses a host can have. * * @author oyving */ public enum HostStatus { - NO_REMARKS, - ALLOWED_TO_BE_DOWN; + /** The services on the host is supposed to be up. */ + NO_REMARKS(false), + + /** The services on the host is allowed to be down. */ + ALLOWED_TO_BE_DOWN(true), + + /** + * Same as ALLOWED_TO_BE_DOWN, but in addition, it is expected + * the host may be removed from its application at any moment. + */ + PERMANENTLY_DOWN(true); + + private final boolean suspended; + + HostStatus(boolean suspended) { this.suspended = suspended; } + public boolean isSuspended() { return suspended; } + public String asString() { return name(); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java index e36f0f70bbd..d2042dc9fd2 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java @@ -14,24 +14,16 @@ import java.util.Set; */ public interface MutableStatusRegistry extends AutoCloseable { - /** - * Returns the status of this application. - */ + /** Returns the status of this application. */ ApplicationInstanceStatus getStatus(); - /** - * Returns the status of the given host. - */ - HostStatus getHostStatus(HostName hostName); + /** Returns the host info of the given host. */ + HostInfo getHostInfo(HostName hostName); - /** - * Returns the set of all suspended hosts for this application. - */ - Set<HostName> getSuspendedHosts(); + /** Returns a snapshot of all host infos for this application. */ + HostInfos getHostInfos(); - /** - * Sets the state for the given host. - */ + /** Sets the state for the given host. */ void setHostState(HostName hostName, HostStatus status); /** diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index 993cddae2b3..e2be5ec7eb6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -57,22 +57,16 @@ public interface StatusService { Set<ApplicationInstanceReference> getAllSuspendedApplications(); /** - * Returns a fresh, but not necessarily consistent mapping from applications to their set of suspended hosts. + * Returns a lambda, which when invoked for an application, returns an up-to-date snapshot of {@link HostInfos host infos}. * - * If the lock for an application is held when this mapping is acquired, new sets returned for that application - * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.) + * <p>Unless the lock for the application is held, the returned snapshot may already be out of date. + * (The snapshot itself is immutable.)</p> */ - Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication(); + Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver(); - /** - * Returns the status of the given application. This is consistent if its lock is held. - */ + /** Returns the status of the given application. This is consistent if its lock is held.*/ ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application); - - /** - * Returns the status of the given host, for the given application. This is consistent if the application's lock is held. - */ - HostStatus getHostStatus(ApplicationInstanceReference application, HostName host); - + /** Get host info for hostname in application. This is consistent if its lock is held. */ + HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index 2d6a9299fcc..b4025167187 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -11,9 +11,9 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; +import com.yahoo.vespa.orchestrator.status.json.WireHostInfo; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; @@ -21,9 +21,10 @@ import org.apache.zookeeper.data.Stat; import javax.inject.Inject; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -42,10 +43,9 @@ public class ZookeeperStatusService implements StatusService { final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service"; final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service"; - final static String HOST_STATUS_CACHE_COUNTER_PATH = "/vespa/host-status-service-cache-counter"; private final Curator curator; - private final CuratorCounter counter; + private final HostInfosCache hostInfosCache; private final Metric metric; private final Timer timer; @@ -55,18 +55,32 @@ public class ZookeeperStatusService implements StatusService { */ private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>(); - /** A cache of hosts allowed to be down. Access only through {@link #getValidCache()}! */ - private final Map<ApplicationInstanceReference, Set<HostName>> hostsDown = new ConcurrentHashMap<>(); - - private volatile long cacheRefreshedAt; - @Inject public ZookeeperStatusService(@Component Curator curator, @Component Metric metric, @Component Timer timer) { this.curator = curator; - this.counter = new CuratorCounter(curator, HOST_STATUS_CACHE_COUNTER_PATH); - this.cacheRefreshedAt = counter.get(); this.metric = metric; this.timer = timer; + + // Insert a cache above some ZooKeeper accesses + this.hostInfosCache = new HostInfosCache(curator, new HostInfosService() { + @Override + public HostInfos getHostInfos(ApplicationInstanceReference application) { + return ZookeeperStatusService.this.getHostInfosFromZk(application); + } + + @Override + public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) { + return ZookeeperStatusService.this.setHostInfoInZk(application, hostName, hostStatus); + } + }); + } + + /** Non-private for testing only. */ + ZookeeperStatusService(Curator curator, Metric metric, Timer timer, HostInfosCache hostInfosCache) { + this.curator = curator; + this.metric = metric; + this.timer = timer; + this.hostInfosCache = hostInfosCache; } @Override @@ -93,13 +107,13 @@ public class ZookeeperStatusService implements StatusService { /** * Cache is checked for freshness when this mapping is created, and may be invalidated again later - * by other users of the cache. Since this function is backed by the cache, any such invalidations + * by other users of the cache. Since this function is backed by the cache, any such invalidation * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it. */ @Override - public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() { - Map<ApplicationInstanceReference, Set<HostName>> suspendedHostsByApplication = getValidCache(); - return application -> suspendedHostsByApplication.computeIfAbsent(application, this::hostsDownFor); + public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() { + hostInfosCache.refreshCache(); + return hostInfosCache::getCachedHostInfos; } @@ -117,6 +131,35 @@ public class ZookeeperStatusService implements StatusService { public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( OrchestratorContext context, ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException { + Runnable onRegistryClose; + + // A multi-application operation, aka batch suspension, will first issue a probe + // then a non-probe. With "large locks", the lock is not release in between - + // no lock is taken on the non-probe. Instead, the release is done on the multi-application + // context close. + if (context.hasLock(applicationInstanceReference)) { + onRegistryClose = () -> {}; + } else { + Runnable unlock = acquireLock(context, applicationInstanceReference); + if (context.registerLockAcquisition(applicationInstanceReference, unlock)) { + onRegistryClose = () -> {}; + } else { + onRegistryClose = unlock; + } + } + + try { + return new ZkMutableStatusRegistry(onRegistryClose, applicationInstanceReference, context.isProbe()); + } catch (Throwable t) { + // In case the constructor throws an exception. + onRegistryClose.run(); + throw t; + } + } + + private Runnable acquireLock(OrchestratorContext context, + ApplicationInstanceReference applicationInstanceReference) + throws UncheckedTimeoutException { ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference); String app = applicationId.application().value() + "." + applicationId.instance().value(); Map<String, String> dimensions = Map.of( @@ -146,66 +189,66 @@ public class ZookeeperStatusService implements StatusService { metric.add(acquireResultMetricName, 1, metricContext); } - Runnable updateLockHoldMetric = () -> { + return () -> { + try { + lock.close(); + } catch (RuntimeException e) { + // We may want to avoid logging some exceptions that may be expected, like when session expires. + log.log(LogLevel.WARNING, + "Failed to close application lock for " + + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", + e); + } + Instant lockReleasedTime = timer.currentTime(); double seconds = durationInSeconds(acquireEndTime, lockReleasedTime); metric.set("orchestrator.lock.hold-latency", seconds, metricContext); }; - - try { - return new ZkMutableStatusRegistry(lock, applicationInstanceReference, context.isProbe(), updateLockHoldMetric); - } catch (Throwable t) { - // In case the constructor throws an exception. - updateLockHoldMetric.run(); - lock.close(); - throw t; - } } private double durationInSeconds(Instant startInstant, Instant endInstant) { return Duration.between(startInstant, endInstant).toMillis() / 1000.0; } - private void setHostStatus(ApplicationInstanceReference applicationInstanceReference, - HostName hostName, - HostStatus status) { - String path = hostAllowedDownPath(applicationInstanceReference, hostName); + /** Returns false if no changes were made. */ + private boolean setHostInfoInZk(ApplicationInstanceReference application, HostName hostname, HostStatus status) { + String path = hostPath(application, hostname); - boolean invalidate = false; - try { - switch (status) { - case NO_REMARKS: - invalidate = deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path); - break; - case ALLOWED_TO_BE_DOWN: - invalidate = createNode_ignoreNodeExistsException(path, "Host already has state ALLOWED_TO_BE_DOWN, path = " + path); - break; - default: - throw new IllegalArgumentException("Unexpected status '" + status + "'."); - } - } catch (Exception e) { - invalidate = true; - throw new RuntimeException(e); + if (status == HostStatus.NO_REMARKS) { + return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path); } - finally { - if (invalidate) { - counter.next(); - hostsDown.remove(applicationInstanceReference); - } + + Optional<HostInfo> currentHostInfo = uncheck(() -> readBytesFromZk(path)).map(WireHostInfo::deserialize); + if (currentHostInfo.isEmpty()) { + Instant suspendedSince = timer.currentTime(); + HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); + byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); + uncheck(() -> curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes)); + } else if (currentHostInfo.get().status() == status) { + return false; + } else { + Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime); + HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); + byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); + uncheck(() -> curator.framework().setData().forPath(path, hostInfoBytes)); } + + return true; } - private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception { + private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) { try { curator.framework().delete().forPath(path); return true; } catch (NoNodeException e) { log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); return false; + } catch (Exception e) { + throw new RuntimeException(e); } } - private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) throws Exception { + private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) { try { curator.framework().create() .creatingParentsIfNeeded() @@ -214,39 +257,54 @@ public class ZookeeperStatusService implements StatusService { } catch (NodeExistsException e) { log.log(LogLevel.DEBUG, debugLogMessageIfExists, e); return false; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Optional<byte[]> readBytesFromZk(String path) throws Exception { + try { + return Optional.of(curator.framework().getData().forPath(path)); + } catch (NoNodeException e) { + return Optional.empty(); } } @Override - public HostStatus getHostStatus(ApplicationInstanceReference applicationInstanceReference, HostName hostName) { - return getValidCache().computeIfAbsent(applicationInstanceReference, this::hostsDownFor) - .contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + public HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName) { + return hostInfosCache.getHostInfos(applicationInstanceReference).getOrNoRemarks(hostName); } - /** Holding an application's lock ensures the cache is up to date for that application. */ - private Map<ApplicationInstanceReference, Set<HostName>> getValidCache() { - long cacheGeneration = counter.get(); - if (counter.get() != cacheRefreshedAt) { - cacheRefreshedAt = cacheGeneration; - hostsDown.clear(); + /** Do not call this directly: should be called behind a cache. */ + private HostInfos getHostInfosFromZk(ApplicationInstanceReference application) { + String hostsRootPath = hostsPath(application); + if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) { + return new HostInfos(); + } else { + List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath)); + Map<HostName, HostInfo> hostInfos = hostnames.stream().collect(Collectors.toMap( + hostname -> new HostName(hostname), + hostname -> { + byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname)); + return WireHostInfo.deserialize(bytes); + })); + return new HostInfos(hostInfos); } - return hostsDown; } - private Set<HostName> hostsDownFor(ApplicationInstanceReference application) { + private <T> T uncheck(SupplierThrowingException<T> supplier) { try { - if (curator.framework().checkExists().forPath(hostsAllowedDownPath(application)) == null) - return Collections.emptySet(); - - return curator.framework().getChildren().forPath(hostsAllowedDownPath(application)) - .stream().map(HostName::new) - .collect(Collectors.toUnmodifiableSet()); - } - catch (Exception e) { + return supplier.get(); + } catch (Exception e) { throw new RuntimeException(e); } } + @FunctionalInterface + interface SupplierThrowingException<T> { + T get() throws Exception; + } + @Override public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference applicationInstanceReference) { try { @@ -259,17 +317,26 @@ public class ZookeeperStatusService implements StatusService { } } - private static String applicationInstancePath(ApplicationInstanceReference applicationInstanceReference) { + static String applicationInstanceReferencePath(ApplicationInstanceReference applicationInstanceReference) { return HOST_STATUS_BASE_PATH + '/' + applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId(); } + private static String applicationPath(ApplicationInstanceReference applicationInstanceReference) { + ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference); + return "/vespa/host-status/" + applicationId.serializedForm(); + } + + private static String hostsPath(ApplicationInstanceReference applicationInstanceReference) { + return applicationPath(applicationInstanceReference) + "/hosts"; + } + private static String hostsAllowedDownPath(ApplicationInstanceReference applicationInstanceReference) { - return applicationInstancePath(applicationInstanceReference) + "/hosts-allowed-down"; + return applicationInstanceReferencePath(applicationInstanceReference) + "/hosts-allowed-down"; } private static String applicationInstanceLock2Path(ApplicationInstanceReference applicationInstanceReference) { - return applicationInstancePath(applicationInstanceReference) + "/lock2"; + return applicationInstanceReferencePath(applicationInstanceReference) + "/lock2"; } private String applicationInstanceSuspendedPath(ApplicationInstanceReference applicationInstanceReference) { @@ -280,21 +347,22 @@ public class ZookeeperStatusService implements StatusService { return hostsAllowedDownPath(applicationInstanceReference) + '/' + hostname.s(); } + private static String hostPath(ApplicationInstanceReference application, HostName hostname) { + return hostsPath(application) + "/" + hostname.s(); + } + private class ZkMutableStatusRegistry implements MutableStatusRegistry { - private final Lock lock; + private final Runnable onClose; private final ApplicationInstanceReference applicationInstanceReference; private final boolean probe; - private final Runnable onLockRelease; - public ZkMutableStatusRegistry(Lock lock, + public ZkMutableStatusRegistry(Runnable onClose, ApplicationInstanceReference applicationInstanceReference, - boolean probe, - Runnable onLockRelease) { - this.lock = lock; + boolean probe) { + this.onClose = onClose; this.applicationInstanceReference = applicationInstanceReference; this.probe = probe; - this.onLockRelease = onLockRelease; } @Override @@ -303,20 +371,20 @@ public class ZookeeperStatusService implements StatusService { } @Override - public HostStatus getHostStatus(HostName hostName) { - return ZookeeperStatusService.this.getHostStatus(applicationInstanceReference, hostName); + public HostInfo getHostInfo(HostName hostName) { + return ZookeeperStatusService.this.getHostInfo(applicationInstanceReference, hostName); } @Override - public Set<HostName> getSuspendedHosts() { - return getValidCache().computeIfAbsent(applicationInstanceReference, ZookeeperStatusService.this::hostsDownFor); + public HostInfos getHostInfos() { + return hostInfosCache.getHostInfos(applicationInstanceReference); } @Override public void setHostState(final HostName hostName, final HostStatus status) { if (probe) return; log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status); - setHostStatus(applicationInstanceReference, hostName, status); + hostInfosCache.setHostStatus(applicationInstanceReference, hostName, status); } @Override @@ -326,31 +394,26 @@ public class ZookeeperStatusService implements StatusService { log.log(LogLevel.INFO, "Setting app " + applicationInstanceReference.asString() + " to status " + applicationInstanceStatus); String path = applicationInstanceSuspendedPath(applicationInstanceReference); - try { - switch (applicationInstanceStatus) { - case NO_REMARKS: - deleteNode_ignoreNoNodeException(path, - "Instance is already in state NO_REMARKS, path = " + path); - break; - case ALLOWED_TO_BE_DOWN: - createNode_ignoreNodeExistsException(path, - "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path); - break; - } - } catch (Exception e) { - throw new RuntimeException(e); + switch (applicationInstanceStatus) { + case NO_REMARKS: + deleteNode_ignoreNoNodeException(path, + "Instance is already in state NO_REMARKS, path = " + path); + break; + case ALLOWED_TO_BE_DOWN: + createNode_ignoreNodeExistsException(path, + "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path); + break; } } @Override public void close() { - onLockRelease.run(); try { - lock.close(); + onClose.run(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. log.log(LogLevel.WARNING, - "Failed to close application lock for " + + "Failed close application lock in " + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", e); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/json/WireHostInfo.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/json/WireHostInfo.java new file mode 100644 index 00000000000..c94384abec5 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/json/WireHostInfo.java @@ -0,0 +1,48 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status.json; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostStatus; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Objects; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Handles serialization/deserialization of HostInfo to/from byte array. + * + * @author hakonhall + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireHostInfo { + private static final ObjectMapper mapper = new ObjectMapper(); + + @JsonProperty("status") public String status; + @JsonProperty("suspendedSince") public Long suspendedSinceInMillis; + + public static HostInfo deserialize(byte[] bytes) { + String serializedString = new String(bytes, StandardCharsets.UTF_8); + WireHostInfo wireHostInfo = uncheck(() -> mapper.readValue(serializedString, WireHostInfo.class)); + return HostInfo.createSuspended(HostStatus.valueOf(Objects.requireNonNull(wireHostInfo.status)), + Instant.ofEpochMilli(Objects.requireNonNull(wireHostInfo.suspendedSinceInMillis))); + } + + public static byte[] serialize(HostInfo hostInfo) { + if (!hostInfo.status().isSuspended()) { + throw new IllegalArgumentException("Serialization of unsuspended status is not supported: " + hostInfo.status()); + } + + WireHostInfo wireHostInfo = new WireHostInfo(); + wireHostInfo.status = hostInfo.status().name(); + wireHostInfo.suspendedSinceInMillis = hostInfo.suspendedSince().get().toEpochMilli(); + + return uncheck(() -> mapper.writeValueAsBytes(wireHostInfo)); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java new file mode 100644 index 00000000000..607894ee104 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java @@ -0,0 +1,59 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.TenantId; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class OrchestratorContextTest { + private final ApplicationInstanceReference application = new ApplicationInstanceReference( + new TenantId("tenant"), + new ApplicationInstanceId("app:dev:us-east-1:default")); + + @Test + public void testLargeLocks() { + var mutable = new Object() { boolean locked = true; }; + Runnable unlock = () -> mutable.locked = false; + + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), true)) { + try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { + assertFalse(probeContext.hasLock(application)); + assertTrue(probeContext.registerLockAcquisition(application, unlock)); + + assertTrue(probeContext.hasLock(application)); + assertTrue(mutable.locked); + } + + try (OrchestratorContext nonProbeContext = rootContext.createSubcontextForSingleAppOp(false)) { + assertTrue(nonProbeContext.hasLock(application)); + assertTrue(mutable.locked); + } + + assertTrue(mutable.locked); + } + assertFalse(mutable.locked); + } + + @Test + public void testLargeLocksDisabled() { + var mutable = new Object() { boolean locked = true; }; + Runnable unlock = () -> mutable.locked = false; + + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), false)) { + try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { + assertFalse(probeContext.hasLock(application)); + assertFalse(probeContext.registerLockAcquisition(application, unlock)); + } + } + + assertTrue(mutable.locked); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 45d4c531898..77ec824da54 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -17,6 +17,8 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; @@ -26,6 +28,7 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; import com.yahoo.vespa.service.monitor.ServiceModel; @@ -36,7 +39,9 @@ import org.mockito.InOrder; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; @@ -55,6 +60,11 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce; /** * Test Orchestrator with a mock backend (the MockCurator) @@ -64,6 +74,7 @@ import static org.mockito.Mockito.spy; public class OrchestratorImplTest { private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private ApplicationId app1; private ApplicationId app2; @@ -88,7 +99,8 @@ public class OrchestratorImplTest { new DummyInstanceLookupService(), 0, new ManualClock(), - applicationApiFactory); + applicationApiFactory, + flagSource); clustercontroller.setAllDummyNodesAsUp(); } @@ -312,6 +324,72 @@ public class OrchestratorImplTest { } @Test + public void testLargeLocks() throws Exception { + flagSource.withBooleanFlag(Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.id(), true); + + var tenantId = new TenantId("tenant"); + var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default"); + var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId); + + var policy = mock(HostedVespaPolicy.class); + var zookeeperStatusService = mock(ZookeeperStatusService.class); + var instanceLookupService = mock(InstanceLookupService.class); + var applicationInstance = mock(ApplicationInstance.class); + var clusterControllerClientFactory = mock(ClusterControllerClientFactory.class); + var clock = new ManualClock(); + var applicationApiFactory = mock(ApplicationApiFactory.class); + var hostStatusRegistry = mock(MutableStatusRegistry.class); + + when(instanceLookupService.findInstanceByHost(any())).thenReturn(Optional.of(applicationInstance)); + when(applicationInstance.reference()).thenReturn(applicationInstanceReference); + when(zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(any(), any())) + .thenReturn(hostStatusRegistry); + when(hostStatusRegistry.getStatus()).thenReturn(NO_REMARKS); + + var orchestrator = new OrchestratorImpl( + policy, + clusterControllerClientFactory, + zookeeperStatusService, + instanceLookupService, + 20, + clock, + applicationApiFactory, + flagSource); + + HostName parentHostname = new HostName("parent.vespa.ai"); + + orchestrator.suspendAll(parentHostname, List.of(parentHostname)); + + ArgumentCaptor<OrchestratorContext> contextCaptor = ArgumentCaptor.forClass(OrchestratorContext.class); + verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(contextCaptor.capture(), any()); + List<OrchestratorContext> contexts = contextCaptor.getAllValues(); + + // First invocation is probe, second is not. + assertEquals(2, contexts.size()); + assertTrue(contexts.get(0).isProbe()); + assertTrue(contexts.get(0).largeLocks()); + assertFalse(contexts.get(1).isProbe()); + assertTrue(contexts.get(1).largeLocks()); + + verify(applicationApiFactory, times(2)).create(any(), any(), any()); + verify(policy, times(2)).grantSuspensionRequest(any(), any()); + verify(instanceLookupService, atLeastOnce()).findInstanceByHost(any()); + verify(hostStatusRegistry, times(2)).getStatus(); + + // Each zookeeperStatusService that is created, is closed. + verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(any(), any()); + verify(hostStatusRegistry, times(2)).close(); + + verifyNoMoreInteractions( + policy, + clusterControllerClientFactory, + zookeeperStatusService, + hostStatusRegistry, + instanceLookupService, + applicationApiFactory); + } + + @Test public void testGetHost() throws Exception { ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); @@ -349,14 +427,16 @@ public class OrchestratorImplTest { lookupService, 0, new ManualClock(), - applicationApiFactory); + applicationApiFactory, + flagSource); orchestrator.setNodeStatus(hostName, HostStatus.ALLOWED_TO_BE_DOWN); Host host = orchestrator.getHost(hostName); assertEquals(reference, host.getApplicationInstanceReference()); assertEquals(hostName, host.getHostName()); - assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostStatus()); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostInfo().status()); + assertTrue(host.getHostInfo().suspendedSince().isPresent()); assertEquals(2, host.getServiceInstances().size()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java index d5734a73de0..ee33a92367b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java @@ -327,7 +327,7 @@ public class ApplicationApiImplTest { private void verifyStorageNodesAllowedToBeDown( ApplicationApi applicationApi, HostName... hostNames) { List<HostName> actualStorageNodes = - applicationApi.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder().stream() + applicationApi.getSuspendedStorageNodesInGroupInReverseClusterOrder().stream() .map(storageNode -> storageNode.hostName()) .collect(Collectors.toList()); assertEquals(Arrays.asList(hostNames), actualStorageNodes); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 8e42ccdc79d..62925dc003e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import org.junit.Test; @@ -75,7 +76,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); @@ -184,7 +185,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp()); @@ -214,7 +215,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(applicationInstance, hostName1, hostName3), - new HashMap<>(), + new HostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertTrue(clusterApi.isStorageCluster()); @@ -254,7 +255,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(application, hostnames.get(0)), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), clusterSize); assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices()); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index 729b3ae79ff..eff222bc074 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.OrchestratorContext; @@ -25,6 +26,8 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -33,11 +36,13 @@ import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.yolean.Exceptions; import java.time.Clock; +import java.time.Instant; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.mockito.Mockito.mock; @@ -48,6 +53,7 @@ class ModelTestUtils { public static final int NUMBER_OF_CONFIG_SERVERS = 3; + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>(); private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>(); @@ -58,14 +64,30 @@ class ModelTestUtils { new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)), 0, new ManualClock(), - applicationApiFactory()); + applicationApiFactory(), + flagSource); ApplicationApiFactory applicationApiFactory() { return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS); } - Map<HostName, HostStatus> getHostStatusMap() { - return hostStatusMap; + HostInfos getHostInfos() { + Instant now = Instant.now(); + + Map<HostName, HostInfo> hostInfosMap = hostStatusMap.entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> { + HostStatus status = entry.getValue(); + if (status == HostStatus.NO_REMARKS) { + return HostInfo.createNoRemarks(); + } else { + return HostInfo.createSuspended(status, now); + } + } + )); + + return new HostInfos(hostInfosMap); } HostName createNode(String name, HostStatus hostStatus) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index b27e37ac034..ed6917a3a4e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -120,7 +120,7 @@ public class HostedVespaPolicyTest { when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); - when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames); + when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames); InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); @@ -136,7 +136,7 @@ public class HostedVespaPolicyTest { order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN); order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN); - order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); + order.verify(applicationApi).getNodesInGroupWith(any()); order.verify(applicationApi).setHostState(context, hostName1, HostStatus.ALLOWED_TO_BE_DOWN); order.verify(applicationApi).setHostState(context, hostName2, HostStatus.ALLOWED_TO_BE_DOWN); order.verify(applicationApi).setHostState(context, hostName3, HostStatus.ALLOWED_TO_BE_DOWN); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index 89f421e9125..80d0af09792 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -155,9 +155,11 @@ public class ApplicationSuspensionResourceTest { return "<services>\n" + " <container version=\"1.0\" jetty=\"true\">\n" + + " <accesslog type=\"disabled\"/>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>10</maxthreads>\n" + " </config>\n" + + " <component id=\"com.yahoo.vespa.flags.InMemoryFlagSource\" bundle=\"flags\" />\n" + " <component id=\"com.yahoo.vespa.curator.mock.MockCurator\" bundle=\"zkfacade\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.status.ZookeeperStatusService\" bundle=\"orchestrator\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" + diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index fec1554396d..dc26c1a3770 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.Host; @@ -36,6 +37,7 @@ import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -90,6 +92,8 @@ public class HostResourceTest { makeServiceClusterSet()))); } + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() { @Override public Optional<ApplicationInstance> findInstanceById( @@ -129,23 +133,23 @@ public class HostResourceTest { } } - private static final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( + private final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory - ); + applicationApiFactory, + flagSource); - private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( + private final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory - ); + applicationApiFactory, + flagSource); private final UriInfo uriInfo = mock(UriInfo.class); @@ -247,7 +251,8 @@ public class HostResourceTest { EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory); + applicationApiFactory, + flagSource); try { HostResource hostResource = new HostResource(alwaysRejectResolver, uriInfo); @@ -267,7 +272,8 @@ public class HostResourceTest { mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory); + applicationApiFactory, + flagSource); try { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver); @@ -343,7 +349,7 @@ public class HostResourceTest { Host host = new Host( hostName, - HostStatus.ALLOWED_TO_BE_DOWN, + HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH), new ApplicationInstanceReference( new TenantId("tenantId"), new ApplicationInstanceId("applicationId")), @@ -353,6 +359,7 @@ public class HostResourceTest { assertEquals("https://foo.com/bar", response.applicationUrl()); assertEquals("hostname", response.hostname()); assertEquals("ALLOWED_TO_BE_DOWN", response.state()); + assertEquals("1970-01-01T00:00:00Z", response.suspendedSince()); assertEquals(1, response.services().size()); assertEquals("clusterId", response.services().get(0).clusterId); assertEquals("configId", response.services().get(0).configId); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfoTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfoTest.java new file mode 100644 index 00000000000..539a10a695d --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfoTest.java @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.orchestrator.status.json.WireHostInfo; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * @author hakonhall + */ +public class HostInfoTest { + private final TestTimer timer = new TestTimer(); + + @Before + public void setUp() { + timer.setMillis(3L); + } + + @Test + public void equality() { + HostInfo info1 = HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, timer.currentTime()); + assertNotEquals(info1, HostInfo.createNoRemarks()); + assertNotEquals(info1, HostInfo.createSuspended(HostStatus.PERMANENTLY_DOWN, timer.currentTime())); + + byte[] serialized = WireHostInfo.serialize(info1); + HostInfo deserialized1 = WireHostInfo.deserialize(serialized); + assertEquals(info1, deserialized1); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfosCacheTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfosCacheTest.java new file mode 100644 index 00000000000..81a160b8636 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/HostInfosCacheTest.java @@ -0,0 +1,56 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Test; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class HostInfosCacheTest { + @Test + public void test() { + MockCurator curator = new MockCurator(); + HostInfosService service = mock(HostInfosService.class); + HostInfosCache cache = new HostInfosCache(curator, service); + + ApplicationInstanceReference application = new ApplicationInstanceReference( + new TenantId("tenantid"), + new ApplicationInstanceId("application:dev:region:default")); + + HostInfos hostInfos = mock(HostInfos.class); + when(service.getHostInfos(application)).thenReturn(hostInfos); + + // cache miss + HostInfos hostInfos1 = cache.getHostInfos(application); + verify(service, times(1)).getHostInfos(any()); + assertSame(hostInfos1, hostInfos); + + // cache hit + HostInfos hostInfos2 = cache.getHostInfos(application); + verify(service, times(1)).getHostInfos(any()); + assertSame(hostInfos2, hostInfos); + + when(service.setHostStatus(any(), any(), any())).thenReturn(true); + boolean modified = cache.setHostStatus(application, new HostName("hostname1"), HostStatus.ALLOWED_TO_BE_DOWN); + verify(service, times(1)).getHostInfos(any()); + assertTrue(modified); + + // cache miss + HostInfos hostInfos3 = cache.getHostInfos(application); + verify(service, times(2)).getHostInfos(any()); + assertSame(hostInfos1, hostInfos); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java new file mode 100644 index 00000000000..8f530f4abf3 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java @@ -0,0 +1,127 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Timer; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.orchestrator.OrchestratorContext; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class ZookeeperStatusService2Test { + private final Curator curator = mock(Curator.class); + private final Timer timer = new TestTimer(); + private final Metric metric = mock(Metric.class); + private final HostInfosCache cache = mock(HostInfosCache.class); + private final ZookeeperStatusService zookeeperStatusService = new ZookeeperStatusService(curator, metric, timer, cache); + + private final OrchestratorContext context = mock(OrchestratorContext.class); + private final InterProcessMutex mutex = mock(InterProcessMutex.class); + private final ApplicationInstanceReference reference = new ApplicationInstanceReference( + new TenantId("tenant"), new ApplicationInstanceId("app:dev:us-east-1:default")); + + @Test + public void verifyLocks() throws Exception { + when(context.isProbe()).thenReturn(true); + when(context.hasLock(any())).thenReturn(false); + when(context.registerLockAcquisition(any(), any())).thenReturn(false); + + when(curator.createMutex(any())).thenReturn(mutex); + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(curator, times(1)).createMutex(any()); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(1)).release(); + verify(context, times(1)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Now the non-probe suspension + + when(context.isProbe()).thenReturn(false); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(mutex, times(2)).acquire(anyLong(), any()); + verify(mutex, times(2)).release(); + ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class); + verify(context, times(2)).hasLock(any()); + verify(context, times(2)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + } + + @Test + public void verifyLargeLocks() throws Exception { + when(context.isProbe()).thenReturn(true); + when(context.hasLock(any())).thenReturn(false); + when(context.registerLockAcquisition(any(), any())).thenReturn(true); + + when(curator.createMutex(any())).thenReturn(mutex); + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(curator, times(1)).createMutex(any()); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(0)).release(); + verify(context, times(1)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Now the non-probe suspension + + when(context.isProbe()).thenReturn(false); + when(context.hasLock(any())).thenReturn(true); + when(context.registerLockAcquisition(any(), any())).thenReturn(false); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + // No (additional) acquire, and no releases. + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(0)).release(); + verify(context, times(2)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Verify the context runnable releases the mutex + ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class); + verify(context, times(1)).registerLockAcquisition(any(), runnableCaptor.capture()); + assertEquals(1, runnableCaptor.getAllValues().size()); + runnableCaptor.getAllValues().forEach(Runnable::run); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(1)).release(); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java index f0f9ebfb13c..12622f22837 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java @@ -96,7 +96,7 @@ public class ZookeeperStatusServiceTest { @Test public void host_state_for_unknown_hosts_is_no_remarks() { assertThat( - zookeeperStatusService.getHostStatus(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1), + zookeeperStatusService.getHostInfo(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1).status(), is(HostStatus.NO_REMARKS)); } @@ -111,13 +111,13 @@ public class ZookeeperStatusServiceTest { .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { //shuffling to catch "clean database" failures for all cases. - for (HostStatus hostStatus: shuffledList(HostStatus.values())) { + for (HostStatus hostStatus: shuffledList(HostStatus.NO_REMARKS, HostStatus.ALLOWED_TO_BE_DOWN)) { for (int i = 0; i < 2; i++) { statusRegistry.setHostState( TestIds.HOST_NAME1, hostStatus); - assertThat(statusRegistry.getHostStatus(TestIds.HOST_NAME1), + assertThat(statusRegistry.getHostInfo(TestIds.HOST_NAME1).status(), is(hostStatus)); } } @@ -182,7 +182,7 @@ public class ZookeeperStatusServiceTest { killSession(curator.framework(), testingServer); //Throws SessionFailedException if the SessionFailRetryLoop has not been closed. - statusRegistry.getHostStatus(TestIds.HOST_NAME1); + statusRegistry.getHostInfo(TestIds.HOST_NAME1); }); assertThat(resultOfZkOperationAfterLockFailure, notHoldsException()); @@ -289,7 +289,8 @@ public class ZookeeperStatusServiceTest { } //TODO: move to vespajlib - private static <T> List<T> shuffledList(T[] values) { + @SafeVarargs + private static <T> List<T> shuffledList(T... values) { //new ArrayList necessary to avoid "write through" behaviour List<T> list = new ArrayList<>(Arrays.asList(values)); Collections.shuffle(list); diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 937431fb7dd..5c8b5b029d2 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -260,7 +260,7 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, << entry.getDocument()->toString(true); } EXPECT_EQ(wanted[wantedIdx].timestamp, entry.getTimestamp()); - size_t serSize = wanted[wantedIdx].doc->serialize()->getLength(); + size_t serSize = wanted[wantedIdx].doc->serialize().size(); EXPECT_EQ(serSize + sizeof(DocEntry), size_t(entry.getSize())); EXPECT_EQ(serSize, size_t(entry.getDocumentSize())); ++wantedIdx; diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 7f6bd835cd5..e35a6a74bde 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -6,9 +6,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> -namespace storage { - -namespace spi { +namespace storage::spi { UpdateResult AbstractPersistenceProvider::update(const Bucket& bucket, Timestamp ts, @@ -66,5 +64,3 @@ AbstractPersistenceProvider::move(const Bucket& source, PartitionId target, Cont } } - -} diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 27197c2adb8..557a9ec2edd 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/bucket.cpp b/persistence/src/vespa/persistence/spi/bucket.cpp index 9265f995a45..ef94519cdb0 100644 --- a/persistence/src/vespa/persistence/spi/bucket.cpp +++ b/persistence/src/vespa/persistence/spi/bucket.cpp @@ -4,8 +4,7 @@ #include <ostream> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { vespalib::string Bucket::toString() const { @@ -30,5 +29,4 @@ operator<<(std::ostream& os, const Bucket& bucket) { return os << bucket.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h index 54760304694..874074d7e24 100644 --- a/persistence/src/vespa/persistence/spi/bucket.h +++ b/persistence/src/vespa/persistence/spi/bucket.h @@ -17,8 +17,7 @@ #include <persistence/spi/types.h> #include <vespa/document/bucket/bucket.h> -namespace storage { -namespace spi { +namespace storage::spi { class Bucket { document::Bucket _bucket; @@ -47,6 +46,4 @@ public: vespalib::asciistream& operator<<(vespalib::asciistream& out, const Bucket& bucket); std::ostream& operator<<(std::ostream& out, const Bucket& bucket); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.cpp b/persistence/src/vespa/persistence/spi/bucketinfo.cpp index e60d6152058..5bef59a65f1 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.cpp +++ b/persistence/src/vespa/persistence/spi/bucketinfo.cpp @@ -3,8 +3,7 @@ #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { BucketInfo::BucketInfo() : _checksum(0), @@ -73,5 +72,4 @@ std::ostream& operator<<(std::ostream& out, const BucketInfo& info) { return out << info.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h index fd4605229f8..827aad48d7f 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.h +++ b/persistence/src/vespa/persistence/spi/bucketinfo.h @@ -8,9 +8,7 @@ #include <persistence/spi/types.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index bc30197978f..567d7ebc1ce 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -12,9 +12,9 @@ namespace storage::spi { ClusterState::ClusterState(const lib::ClusterState& state, uint16_t nodeIndex, const lib::Distribution& distribution) - : _state(new lib::ClusterState(state)), + : _state(std::make_unique<lib::ClusterState>(state)), _nodeIndex(nodeIndex), - _distribution(new lib::Distribution(distribution.serialize())) + _distribution(std::make_unique<lib::Distribution>(distribution.serialize())) { } @@ -26,8 +26,8 @@ void ClusterState::deserialize(vespalib::nbostream& i) { i >> _nodeIndex; i >> distribution; - _state.reset(new lib::ClusterState(clusterState)); - _distribution.reset(new lib::Distribution(distribution)); + _state = std::make_unique<lib::ClusterState>(clusterState); + _distribution = std::make_unique<lib::Distribution>(distribution); } ClusterState::ClusterState(vespalib::nbostream& i) { diff --git a/persistence/src/vespa/persistence/spi/clusterstateimpl.h b/persistence/src/vespa/persistence/spi/clusterstateimpl.h deleted file mode 100644 index 281c37eb9d5..00000000000 --- a/persistence/src/vespa/persistence/spi/clusterstateimpl.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/clusterstate.h> - -namespace storage { - -namespace spi { - -/** - * Used to determine the state of the current node and its buckets. - */ -class ClusterStateImpl : public ClusterState{ -public: - ClusterStateImpl(); - - ClusterStateImpl(const lib::ClusterState& state, - uint16_t nodeIndex, - const lib::Distribution& distribution); - - ClusterStateImpl(vespalib::nbostream& i); - - ClusterStateImpl(const ClusterStateImpl& other); - - ClusterStateImpl& operator=(const ClusterStateImpl& other); - - /** - * Returns true if the given bucket is in the ideal state - * for readiness. - * - * @param b The bucket to check. - */ - bool shouldBeReady(const Bucket& b) const; - - /** - * Returns false if the cluster has been deemed down. This can happen - * if the fleet controller has detected that too many nodes are down - * compared to the complete list of nodes, and deigns the system to be - * unusable. - */ - bool clusterUp() const; - - /** - * Returns false if this node has been set in a state where it should not - * receive external load. - */ - bool nodeUp() const; - - /** - * Returns a serialized form of this object. - */ - void serialize(vespalib::nbostream& o) const; - -private: - std::unique_ptr<lib::ClusterState> _state; - uint16_t _nodeIndex; - std::unique_ptr<lib::Distribution> _distribution; - - void deserialize(vespalib::nbostream&); -}; - -} - -} - diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index 5ce34d5d139..429e2fb9d4e 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -2,8 +2,7 @@ #include "context.h" -namespace storage { -namespace spi { +namespace storage::spi { Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) : _loadType(&loadType), @@ -12,7 +11,6 @@ Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) _readConsistency(ReadConsistency::STRONG) { } -Context::~Context() { } +Context::~Context() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index ca4c79e3005..8c31439ee75 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -29,13 +29,10 @@ #pragma once -#include <persistence/spi/types.h> -#include <vespa/persistence/spi/read_consistency.h> +#include "read_consistency.h" #include <vespa/vespalib/trace/trace.h> -namespace metrics { - class LoadType; -} +namespace metrics { class LoadType; } namespace storage::spi { @@ -62,10 +59,6 @@ public: const LoadType& getLoadType() const { return *_loadType; } Priority getPriority() const { return _priority; } - int getMaxTraceLevel() const { return _trace.getLevel(); } - void addTrace(const vespalib::TraceNode& traceNode) { - _trace.getRoot().addChild(traceNode); - } /** * A read operation might choose to relax its consistency requirements, diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index d482d144d73..f46be6f3a25 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -2,6 +2,7 @@ #include "docentry.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/objects/nbostream.h> #include <sstream> #include <cassert> @@ -10,16 +11,13 @@ namespace storage::spi { DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc) : _timestamp(t), _metaFlags(metaFlags), - _persistedDocumentSize(doc->getSerializedSize()), + _persistedDocumentSize(doc->serialize().size()), _size(_persistedDocumentSize + sizeof(DocEntry)), _documentId(), _document(std::move(doc)) { } -DocEntry::DocEntry(Timestamp t, - int metaFlags, - DocumentUP doc, - size_t serializedDocumentSize) +DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc, size_t serializedDocumentSize) : _timestamp(t), _metaFlags(metaFlags), _persistedDocumentSize(serializedDocumentSize), diff --git a/persistence/src/vespa/persistence/spi/exceptions.cpp b/persistence/src/vespa/persistence/spi/exceptions.cpp index a1b9d57270c..d17c0f90ca0 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.cpp +++ b/persistence/src/vespa/persistence/spi/exceptions.cpp @@ -2,11 +2,8 @@ #include "exceptions.h" -namespace storage { -namespace spi { +namespace storage::spi { VESPA_IMPLEMENT_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/exceptions.h b/persistence/src/vespa/persistence/spi/exceptions.h index 1c434fd8f00..e972e304567 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.h +++ b/persistence/src/vespa/persistence/spi/exceptions.h @@ -3,8 +3,7 @@ #include <vespa/vespalib/util/exceptions.h> -namespace storage { -namespace spi { +namespace storage::spi { /** * Exception used where the cause has already been reported to the user, so @@ -16,6 +15,4 @@ namespace spi { */ VESPA_DEFINE_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/matcher.h b/persistence/src/vespa/persistence/spi/matcher.h index 02dc6db2261..bf989530f35 100644 --- a/persistence/src/vespa/persistence/spi/matcher.h +++ b/persistence/src/vespa/persistence/spi/matcher.h @@ -8,12 +8,11 @@ #pragma once -#include <vespa/persistence/spi/docentry.h> +#include "docentry.h" #include <persistence/spi/documentsubset.h> #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { class Matcher { DocumentSubset _subset; @@ -37,6 +36,4 @@ struct AllMatcher : public Matcher { bool match(const DocEntry&) const { return true; } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.cpp b/persistence/src/vespa/persistence/spi/partitionstate.cpp index 123a82829ef..7cc42742019 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.cpp +++ b/persistence/src/vespa/persistence/spi/partitionstate.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { PartitionState::PartitionState() : _state(UP), @@ -21,7 +20,7 @@ PartitionStateList::PartitionStateList(PartitionId::Type partitionCount) : _states(partitionCount) { } -PartitionStateList::~PartitionStateList() { } +PartitionStateList::~PartitionStateList() = default; PartitionState& PartitionStateList::operator[](PartitionId::Type index) @@ -34,5 +33,4 @@ PartitionStateList::operator[](PartitionId::Type index) return _states[index]; } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.h b/persistence/src/vespa/persistence/spi/partitionstate.h index 6296a70b2c1..e5ce24abbe6 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.h +++ b/persistence/src/vespa/persistence/spi/partitionstate.h @@ -16,8 +16,7 @@ #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { struct PartitionState { enum State { UP, DOWN }; @@ -49,6 +48,4 @@ public: PartitionId size() const { return PartitionId(_states.size()); } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index ec71928baba..61d141c0229 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -2,11 +2,8 @@ #include "persistenceprovider.h" -namespace storage { -namespace spi { +namespace storage::spi { -PersistenceProvider::~PersistenceProvider() { } - -} // spi -} // storage +PersistenceProvider::~PersistenceProvider() = default; +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 96b3d385b87..857630e2606 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -11,9 +11,7 @@ #include "selection.h" #include "clusterstate.h" -namespace document { - class FieldSet; -} +namespace document { class FieldSet; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/providerfactory.h b/persistence/src/vespa/persistence/spi/providerfactory.h index 6b4f0b20ae9..8be143851dd 100644 --- a/persistence/src/vespa/persistence/spi/providerfactory.h +++ b/persistence/src/vespa/persistence/spi/providerfactory.h @@ -8,14 +8,11 @@ #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" -namespace document { - class DocumentTypeRepo; -} +namespace document { class DocumentTypeRepo; } -namespace storage { -namespace spi { +namespace storage::spi { struct ProviderFactory { virtual ~ProviderFactory() {} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.cpp b/persistence/src/vespa/persistence/spi/read_consistency.cpp index cad15a5263d..c3b55e5a261 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.cpp +++ b/persistence/src/vespa/persistence/spi/read_consistency.cpp @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "read_consistency.h" -#include <iostream> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".persistence.spi.read_consistency"); -namespace storage { -namespace spi { +namespace storage::spi { std::ostream& operator<<(std::ostream& os, ReadConsistency consistency) @@ -24,6 +23,4 @@ operator<<(std::ostream& os, ReadConsistency consistency) return os; } -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.h b/persistence/src/vespa/persistence/spi/read_consistency.h index 507e05027cc..d90c43af407 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.h +++ b/persistence/src/vespa/persistence/spi/read_consistency.h @@ -2,10 +2,9 @@ #pragma once #include <iosfwd> -#include <stdint.h> +#include <cstdint> -namespace storage { -namespace spi { +namespace storage::spi { enum class ReadConsistency : uint8_t { /** @@ -28,9 +27,7 @@ enum class ReadConsistency : uint8_t { WEAK }; -std::ostream& -operator<<(std::ostream&, ReadConsistency); +std::ostream& operator<<(std::ostream&, ReadConsistency); -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index 024f5595102..f5131f08b1f 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -9,7 +9,7 @@ namespace storage::spi { Result::Result(const Result &) = default; Result & Result::operator = (const Result &) = default; -Result::~Result() { } +Result::~Result() = default; vespalib::string Result::toString() const { @@ -33,10 +33,10 @@ GetResult::GetResult(Document::UP doc, Timestamp timestamp) _doc(std::move(doc)) { } -GetResult::~GetResult() { } -BucketIdListResult::~BucketIdListResult() { } +GetResult::~GetResult() = default; +BucketIdListResult::~BucketIdListResult() = default; -IterateResult::~IterateResult() { } +IterateResult::~IterateResult() = default; } diff --git a/persistence/src/vespa/persistence/spi/selection.cpp b/persistence/src/vespa/persistence/spi/selection.cpp index 0cd50446488..c7ed98b9d92 100644 --- a/persistence/src/vespa/persistence/spi/selection.cpp +++ b/persistence/src/vespa/persistence/spi/selection.cpp @@ -2,8 +2,7 @@ #include "selection.h" -namespace storage { -namespace spi { +namespace storage::spi { Selection::Selection(const DocumentSelection& docSel) : _documentSelection(docSel), @@ -12,8 +11,7 @@ Selection::Selection(const DocumentSelection& docSel) _timestampSubset() { } -Selection::~Selection() { } +Selection::~Selection() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/selection.h b/persistence/src/vespa/persistence/spi/selection.h index 5c562cabd34..0f809cf1641 100644 --- a/persistence/src/vespa/persistence/spi/selection.h +++ b/persistence/src/vespa/persistence/spi/selection.h @@ -10,12 +10,7 @@ #include "documentselection.h" -namespace storage { -namespace spi { - -class MetaData { - Timestamp timestamp; -}; +namespace storage::spi { class Selection { public: @@ -67,13 +62,6 @@ public: Timestamp getFromTimestamp() const { return _fromTimestamp; } Timestamp getToTimestamp() const { return _toTimestamp; } - - /** - * Regular usage. - */ - bool match(const Document& doc, const MetaData& metaData) const; }; -} // spi -} // storage - +} diff --git a/persistencetypes/src/persistence/spi/types.cpp b/persistencetypes/src/persistence/spi/types.cpp index 00aa95b9707..e746f02e869 100644 --- a/persistencetypes/src/persistence/spi/types.cpp +++ b/persistencetypes/src/persistence/spi/types.cpp @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "types.h" - #include <vespa/vespalib/objects/nbostream.h> -namespace storage { - -namespace spi { +namespace storage::spi { DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(NodeIndex); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(PartitionId); @@ -14,5 +11,3 @@ DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(Timestamp); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(BucketChecksum); } - -} diff --git a/processing/abi-spec.json b/processing/abi-spec.json index 78058f1a8b7..8f77672faec 100644 --- a/processing/abi-spec.json +++ b/processing/abi-spec.json @@ -329,6 +329,10 @@ "public final void set(java.lang.String, java.lang.Object, java.util.Map)", "public final void set(com.yahoo.processing.request.CompoundName, java.lang.Object)", "public final void set(java.lang.String, java.lang.Object)", + "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)", + "public final void clearAll(java.lang.String, java.lang.Object, java.util.Map)", + "public final void clearAll(com.yahoo.processing.request.CompoundName)", + "public final void clearAll(java.lang.String)", "public final boolean getBoolean(com.yahoo.processing.request.CompoundName)", "public final boolean getBoolean(java.lang.String)", "public final boolean getBoolean(com.yahoo.processing.request.CompoundName, boolean)", diff --git a/processing/src/main/java/com/yahoo/processing/execution/Execution.java b/processing/src/main/java/com/yahoo/processing/execution/Execution.java index ee1e9eb39e4..aeeae58543e 100644 --- a/processing/src/main/java/com/yahoo/processing/execution/Execution.java +++ b/processing/src/main/java/com/yahoo/processing/execution/Execution.java @@ -206,7 +206,7 @@ public class Execution { * Creates an empty environment. Only useful for some limited testing */ public static <C extends Processor> Environment<C> createEmpty() { - return new Environment<C>(new ChainRegistry<C>()); + return new Environment<>(new ChainRegistry<>()); } /** @@ -254,7 +254,7 @@ public class Execution { /** * If true, do timing logic, even though trace level is low. */ - private boolean forceTimestamps = false; + private boolean forceTimestamps; /** * Creates an empty root trace with a given level of tracing diff --git a/processing/src/main/java/com/yahoo/processing/request/Properties.java b/processing/src/main/java/com/yahoo/processing/request/Properties.java index 095a0e51ce0..cadc658417b 100644 --- a/processing/src/main/java/com/yahoo/processing/request/Properties.java +++ b/processing/src/main/java/com/yahoo/processing/request/Properties.java @@ -206,8 +206,8 @@ public class Properties implements Cloneable { * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @param context the context used to resolve where the values should be set, or null if none * @throws RuntimeException if no instance in the chain accepted this name-value pair */ @@ -223,8 +223,8 @@ public class Properties implements Cloneable { * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @param context the context used to resolve where the values should be set, or null if none * @throws RuntimeException if no instance in the chain accepted this name-value pair */ @@ -235,8 +235,8 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it by calling set(name,value,null). * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @throws RuntimeException if no instance in the chain accepted this name-value pair */ public final void set(CompoundName name, Object value) { @@ -246,8 +246,8 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it by calling set(name,value,null). * - * @param name the name of the value - * @param value the value to set. Setting a name to null explicitly is legal. + * @param name the name of the property + * @param value the value to set. Setting a property to null clears it. * @throws RuntimeException if no instance in the chain accepted this name-value pair */ public final void set(String name, Object value) { @@ -255,6 +255,56 @@ public class Properties implements Cloneable { } /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * This default implementation forwards to the chained instance or throws + * a RuntimeException if there is not chained instance. + * + * @param name the compound prefix of the properties to clear + * @param context the context used to resolve where the values should be cleared, or null if none + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public void clearAll(CompoundName name, Map<String, String> context) { + if (chained == null) throw new RuntimeException("Property '" + name + + "' was not accepted in this property chain"); + chained.clearAll(name, context); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @param context the context used to resolve where the values should be cleared, or null if none + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(String name, Object value, Map<String, String> context) { + set(new CompoundName(name), value, context); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(CompoundName name) { + clearAll(name, null); + } + + /** + * Sets all properties having this name as a compound prefix to null. + * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab". + * + * @param name the compound prefix of the properties to clear + * @throws RuntimeException if no instance in the chain accepted this name-value pair + */ + public final void clearAll(String name) { + clearAll(new CompoundName(name), Collections.<String,String>emptyMap()); + } + + /** * Gets a property as a boolean - if this value can reasonably be interpreted as a boolean, this will return * the value. Returns false if this property is null. */ @@ -571,14 +621,14 @@ public class Properties implements Cloneable { } } - /** - * Clones a map by deep cloning each value which is cloneable and shallow copying all other values. - */ + /** Clones a map by deep cloning each value which is cloneable and shallow copying all other values. */ public static Map<CompoundName, Object> cloneMap(Map<CompoundName, Object> map) { return cloneHelper.cloneMap(map); } + /** Clones this object if it is clonable, and the clone is public. Returns null if not */ public static Object clone(Object object) { return cloneHelper.clone(object); } + } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index ac3d2327157..53e57fd9c66 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -40,6 +40,8 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) Config::Config(const Config &) = default; Config & Config::operator = (const Config &) = default; +Config::Config(Config &&) noexcept = default; +Config & Config::operator = (Config &&) noexcept = default; Config::~Config() = default; bool diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index fe464736a6b..2f767061f7a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -19,6 +19,8 @@ public: bool fastSearch_ = false, bool huge_ = false); Config(const Config &); Config & operator = (const Config &); + Config(Config &&) noexcept; + Config & operator = (Config &&) noexcept; ~Config(); BasicType basicType() const { return _basicType; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h index 90484f8cc3a..c802730e6fe 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h @@ -17,19 +17,19 @@ class IConstAttributeFunctor { public: virtual void operator()(const IAttributeVector &attributeVector) = 0; - virtual ~IConstAttributeFunctor() { } + virtual ~IConstAttributeFunctor() = default; }; class IAttributeFunctor { public: virtual void operator()(IAttributeVector &attributeVector) = 0; - virtual ~IAttributeFunctor() { } + virtual ~IAttributeFunctor() = default; }; class IAttributeExecutor { public: - virtual ~IAttributeExecutor() { } + virtual ~IAttributeExecutor() = default; virtual void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const = 0; }; diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp index b5fb251eb26..8de2efd8518 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp @@ -6,12 +6,4 @@ namespace search::attribute { -SearchContextParams::SearchContextParams() - : _diversityAttribute(nullptr), - _diversityCutoffGroups(std::numeric_limits<uint32_t>::max()), - _useBitVector(false), - _diversityCutoffStrict(false) -{ -} - } diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h index 922dc4eded6..538e7a01822 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h @@ -3,6 +3,8 @@ #pragma once #include <cstddef> +#include <limits> +#include <cstdint> namespace search::attribute { @@ -14,15 +16,20 @@ class IAttributeVector; class SearchContextParams { private: const IAttributeVector * _diversityAttribute; - size_t _diversityCutoffGroups; + uint32_t _diversityCutoffGroups; bool _useBitVector; bool _diversityCutoffStrict; public: - SearchContextParams(); + SearchContextParams() + : _diversityAttribute(nullptr), + _diversityCutoffGroups(std::numeric_limits<uint32_t>::max()), + _useBitVector(false), + _diversityCutoffStrict(false) + { } bool useBitVector() const { return _useBitVector; } const IAttributeVector * diversityAttribute() const { return _diversityAttribute; } - size_t diversityCutoffGroups() const { return _diversityCutoffGroups; } + uint32_t diversityCutoffGroups() const { return _diversityCutoffGroups; } bool diversityCutoffStrict() const { return _diversityCutoffStrict; } SearchContextParams &useBitVector(bool value) { @@ -33,7 +40,7 @@ public: _diversityAttribute = value; return *this; } - SearchContextParams &diversityCutoffGroups(size_t groups) { + SearchContextParams &diversityCutoffGroups(uint32_t groups) { _diversityCutoffGroups = groups; return *this; } diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index 19d69b0c541..a21cc43572e 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -91,6 +91,11 @@ Schema::Field::Field(const std::vector<vespalib::string> & lines) { } +Schema::Field::Field(const Field &) = default; +Schema::Field & Schema::Field::operator = (const Field &) = default; +Schema::Field::Field(Field &&) noexcept = default; +Schema::Field & Schema::Field::operator = (Field &&) noexcept = default; + Schema::Field::~Field() = default; void @@ -139,6 +144,11 @@ Schema::IndexField::IndexField(const std::vector<vespalib::string> &lines) { } +Schema::IndexField::IndexField(const IndexField &) = default; +Schema::IndexField & Schema::IndexField::operator = (const IndexField &) = default; +Schema::IndexField::IndexField(IndexField &&) noexcept = default; +Schema::IndexField & Schema::IndexField::operator = (IndexField &&) noexcept = default; + void Schema::IndexField::write(vespalib::asciistream & os, vespalib::stringref prefix) const { @@ -178,7 +188,10 @@ Schema::FieldSet::FieldSet(const std::vector<vespalib::string> & lines) : } } -Schema::FieldSet::~FieldSet() { } +Schema::FieldSet::FieldSet(const FieldSet &) = default; +Schema::FieldSet & Schema::FieldSet::operator = (const FieldSet &) = default; + +Schema::FieldSet::~FieldSet() = default; bool Schema::FieldSet::operator==(const FieldSet &rhs) const @@ -206,19 +219,7 @@ Schema::writeToStream(vespalib::asciistream &os, bool saveToDisk) const } } -Schema::Schema() - : _indexFields(), - _attributeFields(), - _summaryFields(), - _fieldSets(), - _importedAttributeFields(), - _indexIds(), - _attributeIds(), - _summaryIds(), - _fieldSetIds(), - _importedAttributeIds() -{ -} +Schema::Schema() = default; Schema::Schema(const Schema & rhs) = default; Schema & Schema::operator=(const Schema & rhs) = default; diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index d71c14c90b1..e17d219d7e8 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -44,6 +44,10 @@ public: * Create this field based on the given config lines. **/ Field(const std::vector<vespalib::string> & lines); + Field(const Field &); + Field & operator = (const Field &); + Field(Field &&) noexcept; + Field & operator = (Field &&) noexcept; virtual ~Field(); @@ -78,6 +82,10 @@ public: public: IndexField(vespalib::stringref name, DataType dt); IndexField(vespalib::stringref name, DataType dt, CollectionType ct); + IndexField(const IndexField &); + IndexField & operator = (const IndexField &); + IndexField(IndexField &&) noexcept; + IndexField & operator = (IndexField &&) noexcept; /** * Create this index field based on the given config lines. **/ @@ -114,6 +122,10 @@ public: public: FieldSet(vespalib::stringref n) : _name(n), _fields() {} + FieldSet(const FieldSet &); + FieldSet & operator =(const FieldSet &); + FieldSet(FieldSet &&) noexcept = default; + FieldSet & operator =(FieldSet &&) noexcept = default; /** * Create this field collection based on the given config lines. diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index f80558a1bc6..8f2be3257fc 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -8,7 +8,6 @@ #include <vespa/metrics/metricmanager.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/programoptions.h> -#include <vespa/vespalib/util/time.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/config/common/exceptions.h> #include <vespa/fastos/app.h> @@ -25,15 +24,21 @@ struct Params std::string identity; std::string serviceidentity; uint64_t subscribeTimeout; + Params(); ~Params(); }; +Params::Params() + : identity(), + serviceidentity(), + subscribeTimeout(60) +{} Params::~Params() = default; class App : public FastOS_Application { private: - void setupSignals(); + static void setupSignals(); Params parseParams(); public: int Main() override; @@ -73,7 +78,7 @@ App::parseParams() vespalib::ProgramOptions parser(_argc, _argv); parser.setSyntaxMessage("proton -- the nextgen search core"); parser.addOption("identity", params.identity, "Node identity and config id"); - std::string empty(""); + std::string empty; parser.addOption("serviceidentity", params.serviceidentity, empty, "Service node identity and config id"); parser.addOption("subscribeTimeout", params.subscribeTimeout, UINT64_C(600000), "Initial config subscribe timeout"); try { @@ -102,7 +107,7 @@ public: ProtonServiceLayerProcess(const config::ConfigUri & configUri, proton::Proton & proton, PersistenceProvider *downPersistence); - ~ProtonServiceLayerProcess() { shutdown(); } + ~ProtonServiceLayerProcess() override { shutdown(); } void shutdown() override; void setupProvider() override; @@ -126,7 +131,7 @@ ProtonServiceLayerProcess::ProtonServiceLayerProcess(const config::ConfigUri & downPersistence) : ServiceLayerProcess(configUri), _proton(proton), - _metricManager(0), + _metricManager(nullptr), _downPersistence(downPersistence) { if (!downPersistence) { @@ -143,7 +148,7 @@ ProtonServiceLayerProcess::shutdown() void ProtonServiceLayerProcess::setupProvider() { - if (_metricManager != 0) { + if (_metricManager != nullptr) { _context.getComponentRegister().setMetricManager(*_metricManager); } } @@ -270,6 +275,9 @@ App::Main() } catch (const vespalib::NetworkSetupFailureException & e) { LOG(warning, "Network failure: '%s'", e.what()); return 1; + } catch (const config::InvalidConfigException & e) { + LOG(warning, "Invalid config failure: '%s'", e.what()); + return 1; } catch (const vespalib::IllegalStateException & e) { LOG(error, "Unknown IllegalStateException: '%s'", e.what()); throw; diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index ba0d2047b88..4056e49e37c 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -331,7 +331,7 @@ public: LOG(info, "putHandler(%s)", itr->first.toString().c_str()); IPersistenceHandler::SP proxy( new PersistenceHandlerProxy(itr->second)); - putHandler(itr->second->getBucketSpace(), itr->first, proxy); + putHandler(getWLock(), itr->second->getBucketSpace(), itr->first, proxy); } } @@ -343,7 +343,7 @@ public: const DocumentDBMap &docDbs = _docDbRepo->getDocDbs(); for (DocumentDBMap::const_iterator itr = docDbs.begin(); itr != docDbs.end(); ++itr) { - IPersistenceHandler::SP proxy(removeHandler(itr->second->getBucketSpace(), itr->first)); + IPersistenceHandler::SP proxy(removeHandler(getWLock(), itr->second->getBucketSpace(), itr->first)); } } diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index ca8274aab00..54fc072c2b1 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -298,7 +298,7 @@ public: { } virtual RPC::Result receive(const Packet &packet) override { - vespalib::nbostream_longlivedbuf handle(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(packet.getHandle().data(), packet.getHandle().size()); try { while (handle.size() > 0) { Packet::Entry entry; diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index e5ac6ed15b0..6d7bd243e71 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -168,7 +168,7 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock, f1.timeOfDoom, os.c_str(), os.size()); + GroupingContext context(f1.clock, f1.timeOfDoom, os.data(), os.size()); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -226,7 +226,7 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) { context.serialize(); vespalib::nbostream & res(context.getResult()); EXPECT_EQUAL(res.size(), os.size()); - ASSERT_TRUE(memcmp(res.c_str(), os.c_str(), res.size()) == 0); + ASSERT_TRUE(memcmp(res.data(), os.data(), res.size()) == 0); } TEST_F("testGroupingManager", DoomFixture()) { diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index c6d49c479c4..44ff8cb925e 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -215,7 +215,7 @@ struct SequentialAttributeManager { mgr.addInitializedAttributes(initializer.getInitializedAttributes()); } - ~SequentialAttributeManager() {} + ~SequentialAttributeManager() = default; }; struct DummyInitializerTask : public InitializerTask @@ -262,23 +262,33 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria initializer::TaskRunner taskRunner(executor); taskRunner.runTask(initializer); } -ParallelAttributeManager::~ParallelAttributeManager() {} +ParallelAttributeManager::~ParallelAttributeManager() = default; TEST_F("require that attributes are added", Fixture) { - EXPECT_TRUE(f.addAttribute("a1").get() != NULL); - EXPECT_TRUE(f.addAttribute("a2").get() != NULL); + EXPECT_TRUE(f.addAttribute("a1").get() != nullptr); + EXPECT_TRUE(f.addAttribute("a2").get() != nullptr); EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName()); EXPECT_EQUAL("a1", (*f._m.getAttributeReadGuard("a1", true))->getName()); EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName()); EXPECT_EQUAL("a2", (*f._m.getAttributeReadGuard("a2", true))->getName()); EXPECT_TRUE(!f._m.getAttribute("not")->valid()); + + auto rv = f._m.readable_attribute_vector("a1"); + ASSERT_TRUE(rv.get() != nullptr); + EXPECT_EQUAL("a1", rv->makeReadGuard(true)->attribute()->getName()); + + rv = f._m.readable_attribute_vector("a2"); + ASSERT_TRUE(rv.get() != nullptr); + EXPECT_EQUAL("a2", rv->makeReadGuard(true)->attribute()->getName()); + + EXPECT_TRUE(f._m.readable_attribute_vector("not_valid").get() == nullptr); } TEST_F("require that predicate attributes are added", Fixture) { EXPECT_TRUE(f._m.addAttribute({"p1", AttributeUtils::getPredicateConfig()}, - createSerialNum).get() != NULL); + createSerialNum).get() != nullptr); EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName()); EXPECT_EQUAL("p1", (*f._m.getAttributeReadGuard("p1", true))->getName()); } @@ -376,7 +386,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); EXPECT_EQUAL(1u, a1->getNumDocs()); - PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = static_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -396,7 +406,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); // loaded EXPECT_EQUAL(2u, a1->getNumDocs()); - PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = static_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -746,7 +756,7 @@ TEST_F("require that we can acquire exclusive read access to attribute", Fixture EXPECT_TRUE(noneAccessor.get() == nullptr); } -TEST_F("require that imported attributes are exposed via attribute context together vi regular attributes", Fixture) +TEST_F("require that imported attributes are exposed via attribute context together with regular attributes", Fixture) { f.addAttribute("attr"); f.addImportedAttribute("imported"); @@ -767,6 +777,17 @@ TEST_F("require that imported attributes are exposed via attribute context toget EXPECT_EQUAL("imported", all[1]->getName()); } +TEST_F("imported attributes are transparently returned from readable_attribute_vector", Fixture) +{ + f.addAttribute("attr"); + f.addImportedAttribute("imported"); + f.setImportedAttributes(); + auto av = f._m.readable_attribute_vector("imported"); + ASSERT_TRUE(av); + auto g = av->makeReadGuard(false); + EXPECT_EQUAL("imported", g->attribute()->getName()); +} + TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) { AVConfig generic_tensor(BasicType::TENSOR); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 14b72c9d8f8..edb5a07b059 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -20,6 +20,7 @@ #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> @@ -588,8 +589,8 @@ struct FilterFixture TEST_F("require that filter attribute manager can filter attributes", FilterFixture) { - EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL); - EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL); + EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr); + EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr); std::vector<AttributeGuard> attrs; f._filterMgr.getAttributeList(attrs); EXPECT_EQUAL(1u, attrs.size()); @@ -607,6 +608,16 @@ TEST_F("require that filter attribute manager can return flushed serial number", EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); } +TEST_F("readable_attribute_vector filters attributes", FilterFixture) +{ + auto av = f._filterMgr.readable_attribute_vector("a2"); + ASSERT_TRUE(av); + EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName()); + + av = f._filterMgr.readable_attribute_vector("a1"); + EXPECT_FALSE(av); +} + namespace { Tensor::UP make_tensor(const TensorSpec &spec) { diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index da5a8b31671..76dcf53c51d 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/log/log.h> LOG_SETUP("attributes_state_explorer_test"); diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index 5b1fa3ed4bf..744159962ad 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -75,7 +75,9 @@ makeDocTypeRepo() addField("aaa", Array(DataType::T_INT)). addField("aaw", Wset(DataType::T_INT)). addField("ab", DataType::T_INT). - addField("ae", DataType::T_INT)); + addField("ae", DataType::T_INT)). + imported_field("my_imported_field"). + imported_field("my_missing_imported_field"); builder.document(doc_type_id + 1, type_name_2, Struct(header_name_2), Struct(body_name_2). addField("ic", DataType::T_STRING). @@ -150,6 +152,9 @@ TestFixture::TestFixture() _amgr.addAttribute("aaa", AttributeFactory::createAttribute("aaa", { BasicType::INT32 , CollectionType::ARRAY})); _amgr.addAttribute("aaw", AttributeFactory::createAttribute("aaw", { BasicType::INT32 , CollectionType::WSET})); _amgr.addAttribute("ae", AttributeFactory::createAttribute("ae", { BasicType::INT32 })); + // We "fake" having an imported attribute to avoid having to set up reference attributes, mappings etc. + // This is fine since the attribute manager already abstracts away if an attribute is imported or not. + _amgr.addAttribute("my_imported_field", AttributeFactory::createAttribute("my_imported_field", { BasicType::INT32 })); _repoUP = makeDocTypeRepo(); } @@ -170,9 +175,9 @@ TestFixture::testParse(const string &selection) select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.what()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() != NULL); + ASSERT_TRUE(select.get() != nullptr); } @@ -189,9 +194,9 @@ TestFixture::testParseFail(const string &selection) select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.getMessage().c_str()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() == NULL); + ASSERT_TRUE(select.get() == nullptr); } @@ -208,15 +213,15 @@ TestFixture::testPrune(const string &selection, const string &exp, const string select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.what()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() != NULL); + ASSERT_TRUE(select.get() != nullptr); std::ostringstream os; select->print(os, true, ""); LOG(info, "ParseTree: '%s'", os.str().c_str()); const DocumentType *docType = repo.getDocumentType(docTypeName); - ASSERT_TRUE(docType != NULL); - Document::UP emptyDoc(new Document(*docType, document::DocumentId("id:ns:" + docTypeName + "::1"))); + ASSERT_TRUE(docType != nullptr); + auto emptyDoc = std::make_unique<Document>(*docType, document::DocumentId("id:ns:" + docTypeName + "::1")); emptyDoc->setRepo(repo); SelectPruner pruner(docTypeName, &_amgr, *emptyDoc, repo, _hasFields, _hasDocuments); pruner.process(*select); @@ -788,6 +793,29 @@ TEST_F("Test that field values are invalid when disabling document access", Test "test.aa == 4 and test.ae == 5 and invalid"); } +TEST_F("Imported fields with matching attribute names are supported", TestFixture) +{ + f.testPrune("test.my_imported_field > 0", + "test.my_imported_field > 0"); +} + +// Edge case: document type reconfigured but attribute not yet visible in Proton +TEST_F("Imported fields without matching attribute are mapped to constant NullValue", TestFixture) +{ + f.testPrune("test.my_missing_imported_field != test.aa", "null != test.aa"); + // Simplified to -> "null != null" -> "false" + f.testPrune("test.my_missing_imported_field != null", "false"); + // Simplified to -> "null > 0" -> "invalid", as null is not well-defined + // for operators other than (in-)equality. + f.testPrune("test.my_missing_imported_field > 0", "invalid"); +} + +TEST_F("Complex imported field references return Invalid", TestFixture) +{ + f.testPrune("test.my_imported_field.foo", "invalid"); + f.testPrune("test.my_imported_field[123]", "invalid"); + f.testPrune("test.my_imported_field{foo}", "invalid"); +} } // namespace diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 3e86e703e1e..c15fd2ccfc1 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -197,9 +197,9 @@ public: _dummy(), _spec(TEST_PATH("")), _configMgr(_spec, getDocTypeName()), - _documenttypesConfig(new DocumenttypesConfig()), + _documenttypesConfig(std::make_shared<DocumenttypesConfig>()), _repo(repo), - _tuneFileDocumentDB(new TuneFileDocumentDB()), + _tuneFileDocumentDB(std::make_shared<TuneFileDocumentDB>()), _hwInfo(), _ddb(), _aw(), diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 48ab09ffcb2..607a644a302 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -48,43 +48,51 @@ TEST("requireThatAverageTimesAreRecorded") { EXPECT_APPROX(0.0, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(0u, stats.matchTimeCount()); EXPECT_EQUAL(0u, stats.groupingTimeCount()); EXPECT_EQUAL(0u, stats.rerankTimeCount()); EXPECT_EQUAL(0u, stats.queryCollateralTimeCount()); + EXPECT_EQUAL(0u, stats.querySetupTimeCount()); EXPECT_EQUAL(0u, stats.queryLatencyCount()); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyAvg(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.02, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.2, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(4.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(4.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.queryLatencyAvg(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) .queryCollateralTime(10.0) + .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) .queryCollateralTime(10.0).queryCollateralTime(6.0) + .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.03, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(6.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(6.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(4u, stats.matchTimeCount()); EXPECT_EQUAL(4u, stats.groupingTimeCount()); EXPECT_EQUAL(4u, stats.rerankTimeCount()); EXPECT_EQUAL(4u, stats.queryCollateralTimeCount()); + EXPECT_EQUAL(4u, stats.querySetupTimeCount()); EXPECT_EQUAL(4u, stats.queryLatencyCount()); } @@ -94,53 +102,63 @@ TEST("requireThatMinMaxTimesAreRecorded") { EXPECT_APPROX(0.0, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.0, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMax(), 0.00001); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.01, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMax(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.03, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(6.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(6.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyMax(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) .queryCollateralTime(10.0) + .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) .queryCollateralTime(10.0).queryCollateralTime(6.0) + .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.05, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(2.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(10.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(10.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(5.0, stats.queryLatencyMax(), 0.00001); } diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 8c4bf0d55b0..95ab43dbcba 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -616,7 +616,7 @@ TEST("require that grouping is performed (multi-threaded)") { Grouping grequest; grequest.setRoot(Group().addResult(SumAggregationResult().setExpression(createAttr()))); grequest.serialize(os); - request->groupSpec.assign(buf.c_str(), buf.c_str() + buf.size()); + request->groupSpec.assign(buf.data(), buf.data() + buf.size()); } SearchReply::UP reply = world.performSearch(request, threads); { diff --git a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp index 109a4cc7a25..c3338a973c4 100644 --- a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp +++ b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp @@ -36,7 +36,7 @@ private: void insert_tensor_in_properties(const vespalib::string& tensor_name, const Value& tensor_value) { vespalib::nbostream stream; DefaultTensorEngine::ref().encode(tensor_value, stream); - _props.add(tensor_name, vespalib::stringref(stream.c_str(), stream.size())); + _props.add(tensor_name, vespalib::stringref(stream.data(), stream.size())); } public: diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index ef12b694187..35590cc68f6 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -46,23 +46,31 @@ DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>( DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>()); + +void +assertHandler(const IPersistenceHandler::SP & lhs, const IPersistenceHandler * rhs) +{ + EXPECT_EQUAL(lhs.get(), rhs); +} + void assertHandler(const IPersistenceHandler::SP &lhs, const IPersistenceHandler::SP &rhs) { EXPECT_EQUAL(lhs.get(), rhs.get()); } +template <typename T> void -assertNullHandler(const IPersistenceHandler::SP &handler) +assertNullHandler(const T & handler) { - EXPECT_TRUE(handler.get() == nullptr); + EXPECT_TRUE(! handler); } void -assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, const HandlerSnapshot::UP &snapshot) +assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, HandlerSnapshot snapshot) { - EXPECT_EQUAL(exp.size(), snapshot->size()); - auto &sequence = snapshot->handlers(); + EXPECT_EQUAL(exp.size(), snapshot.size()); + auto &sequence = snapshot.handlers(); for (size_t i = 0; i < exp.size() && sequence.valid(); ++i, sequence.next()) { EXPECT_EQUAL(exp[i].get(), sequence.get()); } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 569b36a425d..19d9d41c3e4 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -404,8 +404,8 @@ struct SimpleFixture { engine(_owner, _writeFilter, -1, false), hset() { - engine.putHandler(makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1); - engine.putHandler(bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2); + engine.putHandler(engine.getWLock(), makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1); + engine.putHandler(engine.getWLock(), bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2); } SimpleFixture() : SimpleFixture(makeBucketSpace()) diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index fc12cde57af..30a87bd216e 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -36,6 +36,7 @@ #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> @@ -111,6 +112,8 @@ vespalib::string tensor_spec("tensor(x{})"); std::unique_ptr<Tensor> static_tensor = makeTensor<Tensor>(TensorSpec(tensor_spec).add({{"x", "1"}}, 1.5)); std::unique_ptr<Tensor> dynamic_tensor = makeTensor<Tensor>(TensorSpec(tensor_spec).add({{"x", "2"}}, 3.5)); const char zcurve_field[] = "position_field_zcurve"; +const char position_array_field[] = "position_array"; +const char zcurve_array_field[] = "position_array_zcurve"; const char dyn_field_p[] = "dynamic predicate field"; const char dyn_arr_field_i[] = "dynamic int array field"; const char dyn_arr_field_d[] = "dynamic double array field"; @@ -153,7 +156,7 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { return std::move(_testDoc); } const DocumentType *doc_type = r.getDocumentType(doc_type_name); - Document::UP doc(new Document(*doc_type, doc_id)); + auto doc = std::make_unique<Document>(*doc_type, doc_id); ASSERT_TRUE(doc.get()); doc->set(static_field, static_value); doc->set(dyn_field_i, static_value); @@ -208,10 +211,11 @@ document::DocumenttypesConfig getRepoConfig() { .addField(dyn_wset_field_d, Wset(document::DataType::T_DOUBLE)) .addField(dyn_wset_field_s, Wset(document::DataType::T_STRING)) .addField(dyn_wset_field_n, Wset(document::DataType::T_FLOAT)) - .addField(position_field, - PositionDataType::getInstance().getId()) + .addField(position_field, PositionDataType::getInstance().getId()) .addTensorField(dyn_field_tensor, tensor_spec) - .addField(zcurve_field, document::DataType::T_LONG)); + .addField(zcurve_field, document::DataType::T_LONG) + .addField(position_array_field, Array(PositionDataType::getInstance().getId())) + .addField(zcurve_array_field, Array(document::DataType::T_LONG))); return builder.config(); } @@ -359,6 +363,8 @@ struct Fixture { dyn_arr_field_s, dyn_value_s, DataType::STRING, ct); addAttribute<FloatingPointAttribute>( dyn_arr_field_n, DataType::FLOAT, ct); + addAttribute<IntegerAttribute>( + zcurve_array_field, dynamic_zcurve_value, DataType::INT64, ct); ct = schema::CollectionType::WEIGHTEDSET; addAttribute<IntegerAttribute>( dyn_wset_field_i, dyn_value_i, DataType::INT32, ct); @@ -484,25 +490,51 @@ void verify_position_field_has_expected_values(Fixture& f) { FieldValue::UP value = doc->getValue(position_field); ASSERT_TRUE(value.get()); - StructFieldValue *position = dynamic_cast<StructFieldValue *>(value.get()); + const auto *position = dynamic_cast<StructFieldValue *>(value.get()); ASSERT_TRUE(position); FieldValue::UP x = position->getValue(PositionDataType::FIELD_X); FieldValue::UP y = position->getValue(PositionDataType::FIELD_Y); - EXPECT_EQUAL(-123096000, static_cast<IntFieldValue&>(*x).getValue()); - EXPECT_EQUAL(49401000, static_cast<IntFieldValue&>(*y).getValue()); + EXPECT_EQUAL(-123096000, dynamic_cast<IntFieldValue&>(*x).getValue()); + EXPECT_EQUAL(49401000, dynamic_cast<IntFieldValue&>(*y).getValue()); checkFieldValue<LongFieldValue>(doc->getValue(zcurve_field), dynamic_zcurve_value); } -TEST_F("require that position fields are regenerated from zcurves", Fixture) { +TEST_F("require that single value position fields are regenerated from zcurves", Fixture) { verify_position_field_has_expected_values(f); } -TEST_F("zcurve attribute is authoritative for position field existence", Fixture) { +TEST_F("zcurve attribute is authoritative for single value position field existence", Fixture) { f.doc_store._set_position_struct_field = false; verify_position_field_has_expected_values(f); } +TEST_F("require that array position field value is generated from zcurve array attribute", Fixture) { + DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); + Document::UP doc = f._retriever->getDocument(meta_data.lid); + ASSERT_TRUE(doc.get()); + FieldValue::UP value = doc->getValue(position_array_field); + ASSERT_TRUE(value.get()); + const auto* array_value = dynamic_cast<const document::ArrayFieldValue*>(value.get()); + ASSERT_TRUE(array_value != nullptr); + ASSERT_EQUAL(array_value->getNestedType(), document::PositionDataType::getInstance()); + // Should have two elements prepopulated + ASSERT_EQUAL(2u, array_value->size()); + for (uint32_t i = 0; i < array_value->size(); ++i) { + // Magic index-specific value set by collection fixture code. + int64_t zcurve_at_pos = ((i == 0) ? dynamic_zcurve_value + 1 : dynamic_zcurve_value); + int32_t zx, zy; + vespalib::geo::ZCurve::decode(zcurve_at_pos, &zx, &zy); + + const auto *position = dynamic_cast<const StructFieldValue*>(&(*array_value)[i]); + ASSERT_TRUE(position != nullptr); + FieldValue::UP x = position->getValue(PositionDataType::FIELD_X); + FieldValue::UP y = position->getValue(PositionDataType::FIELD_Y); + EXPECT_EQUAL(zx, dynamic_cast<IntFieldValue&>(*x).getValue()); + EXPECT_EQUAL(zy, dynamic_cast<IntFieldValue&>(*y).getValue()); + } +} + TEST_F("require that non-existing lid returns null pointer", Fixture) { Document::UP doc = f._retriever->getDocument(0); ASSERT_FALSE(doc.get()); diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 96096c0401f..ca48fb773d8 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -105,7 +105,7 @@ RemoveOperationContext::RemoveOperationContext(search::SerialNum serial) str(), packet() { op.serialize(str); - ConstBufferRef buf(str.c_str(), str.wp()); + ConstBufferRef buf(str.data(), str.wp()); packet.reset(new Packet()); packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf)); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index d123a5711ac..eb4f1f6dc89 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -397,8 +397,8 @@ writefilter.memorylimit double default = 0.8 ## put and update portion of feed is blocked. writefilter.disklimit double default = 0.8 -## Interval between sampling of disk and memory usage. Default is 60 seconds. -writefilter.sampleinterval double default = 60.0 +## Interval between sampling of disk and memory usage. Default is 20 seconds. +writefilter.sampleinterval double default = 20.0 ## The size of the disk partition (in bytes) on which proton basedir is located. ## If set to 0, the disk size is sampled by looking at the filesystem space info. diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp index 20e6798c79b..bee80e77bd6 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp @@ -16,10 +16,10 @@ AttributeSpec::AttributeSpec(const AttributeSpec &) = default; AttributeSpec & AttributeSpec::operator=(const AttributeSpec &) = default; -AttributeSpec::AttributeSpec(AttributeSpec &&) = default; +AttributeSpec::AttributeSpec(AttributeSpec &&) noexcept = default; AttributeSpec & -AttributeSpec::operator=(AttributeSpec &&) = default; +AttributeSpec::operator=(AttributeSpec &&) noexcept = default; AttributeSpec::~AttributeSpec() = default; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h index 6c720ff5792..b6d6fc9963b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h @@ -16,12 +16,11 @@ private: vespalib::string _name; search::attribute::Config _cfg; public: - AttributeSpec(const vespalib::string &name, - const search::attribute::Config &cfg); + AttributeSpec(const vespalib::string &name, const search::attribute::Config &cfg); AttributeSpec(const AttributeSpec &); AttributeSpec & operator=(const AttributeSpec &); - AttributeSpec(AttributeSpec &&); - AttributeSpec & operator=(AttributeSpec &&); + AttributeSpec(AttributeSpec &&) noexcept; + AttributeSpec & operator=(AttributeSpec &&) noexcept; ~AttributeSpec(); const vespalib::string &getName() const { return _name; } const search::attribute::Config &getConfig() const { return _cfg; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 34e9ba2c145..74e3e903540 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -62,11 +62,11 @@ AttributeWriter::WriteContext::WriteContext(ExecutorId executorId) } -AttributeWriter::WriteContext::WriteContext(WriteContext &&rhs) = default; +AttributeWriter::WriteContext::WriteContext(WriteContext &&rhs) noexcept = default; AttributeWriter::WriteContext::~WriteContext() = default; -AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) = default; +AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) noexcept = default; void AttributeWriter::WriteContext::add(AttributeVector &attr) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 4ea7f3fda6c..9e5d8f4ce5d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -48,9 +48,9 @@ public: bool _hasStructFieldAttribute; public: WriteContext(ExecutorId executorId); - WriteContext(WriteContext &&rhs); + WriteContext(WriteContext &&rhs) noexcept; ~WriteContext(); - WriteContext &operator=(WriteContext &&rhs); + WriteContext &operator=(WriteContext &&rhs) noexcept; void buildFieldPaths(const DocumentType &docType); void add(AttributeVector &attr); ExecutorId getExecutorId() const { return _executorId; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index e39061a8389..02bf4f4d7cc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> @@ -613,4 +614,14 @@ AttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> _importedAttributes = std::move(attributes); } +std::shared_ptr<search::attribute::ReadableAttributeVector> +AttributeManager::readable_attribute_vector(const string& name) const +{ + auto attribute = findAttribute(name); + if (attribute || !_importedAttributes) { + return attribute; + } + return _importedAttributes->get(name); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index cf60a0a41e9..881452c6b3d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -178,6 +178,8 @@ public: void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); } + + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index fd1f95e13ba..aee68457b62 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -231,4 +231,13 @@ FilterAttributeManager::getImportedAttributes() const throw vespalib::IllegalArgumentException("Not implemented"); } +std::shared_ptr<search::attribute::ReadableAttributeVector> +FilterAttributeManager::readable_attribute_vector(const string& name) const +{ + if (acceptAttribute(name)) { + return _mgr->readable_attribute_vector(name); + } + return {}; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 099bb2e84ba..0bd04b8f0a5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -54,6 +54,7 @@ public: ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override; + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index 3ced0f509b5..e80543368d4 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -119,6 +119,19 @@ BucketDB::cachedGet(const BucketId &bucketId) const return get(bucketId); } +storage::spi::BucketInfo +BucketDB::cachedGetBucketInfo(const BucketId &bucketId) const +{ + if (isCachedBucket(bucketId)) { + return _cachedBucketState; + } + auto itr = _map.find(bucketId); + if (itr != _map.end()) { + return itr->second; + } + return BucketState(); +} + bool BucketDB::hasBucket(const BucketId &bucketId) const { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index e64cc53d4a2..05388931e20 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -51,6 +51,7 @@ public: void cacheBucket(const BucketId &bucketId); void uncacheBucket(); bool isCachedBucket(const BucketId &bucketId) const; + storage::spi::BucketInfo cachedGetBucketInfo(const BucketId &bucketId) const; BucketState cachedGet(const BucketId &bucketId) const; bool hasBucket(const BucketId &bucketId) const; void getBuckets(BucketId::List & buckets) const; diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index ab3b0588f3f..c516bbfc06c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -4,6 +4,7 @@ #include "selectcontext.h" #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> @@ -31,14 +32,10 @@ using vespalib::make_string; AttributeFieldValueNode:: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const std::shared_ptr<search::AttributeVector> &attribute) + uint32_t attr_guard_index) : FieldValueNode(doctype, field), - _attribute(attribute) + _attr_guard_index(attr_guard_index) { - const AttributeVector &v(*_attribute); - // Only handle single value attribute vectors for now - assert(v.getCollectionType() == CollectionType::SINGLE); - (void) v; } @@ -46,10 +43,10 @@ std::unique_ptr<document::select::Value> AttributeFieldValueNode:: getValue(const Context &context) const { - const SelectContext &sc(static_cast<const SelectContext &>(context)); + const auto &sc(static_cast<const SelectContext &>(context)); uint32_t docId(sc._docId); assert(docId != 0u); - const AttributeVector &v(*_attribute); + const auto& v = sc.guarded_attribute_at_index(_attr_guard_index); if (v.isUndefined(docId)) { return std::make_unique<NullValue>(); } @@ -86,10 +83,10 @@ getValue(const Context &context) const case BasicType::PREDICATE: case BasicType::TENSOR: case BasicType::REFERENCE: - throw new IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", - v.getName().c_str(), v.getInternalBasicType().asString())); + throw IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", + v.getName().c_str(), BasicType(v.getBasicType()).asString())); case BasicType::MAX_TYPE: - throw new IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType())); + throw IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType())); } return std::make_unique<NullValue>();; @@ -106,7 +103,7 @@ AttributeFieldValueNode::traceValue(const Context &context, std::ostream& out) c document::select::ValueNode::UP AttributeFieldValueNode::clone() const { - return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute)); + return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attr_guard_index)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h index 9ac6ce0e32b..89d1dac321b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h @@ -3,18 +3,19 @@ #include <vespa/document/select/valuenodes.h> -namespace search { class AttributeVector; } +namespace search { class ReadableAttributeVector; } namespace proton { class AttributeFieldValueNode : public document::select::FieldValueNode { using Context = document::select::Context; - std::shared_ptr<search::AttributeVector> _attribute; + uint32_t _attr_guard_index; public: + // Precondition: attribute must be of a single-value type. AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const std::shared_ptr<search::AttributeVector> &attribute); + uint32_t attr_guard_index); std::unique_ptr<document::select::Value> getValue(const Context &context) const override; std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override; diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index d007b030e6b..21f757fba79 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -7,6 +7,7 @@ #include "selectpruner.h" #include <vespa/document/select/parser.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/iattributemanager.h> namespace proton { @@ -24,7 +25,7 @@ namespace { class AttrVisitor : public document::select::CloningVisitor { public: - typedef std::map<vespalib::string, uint32_t> AttrMap; + using AttrMap = std::map<vespalib::string, uint32_t>; AttrMap _amap; const search::IAttributeManager &_amgr; @@ -62,7 +63,7 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At AttrVisitor::~AttrVisitor() = default; -bool isSingleValueThatWEHandle(BasicType type) { +bool isSingleValueThatWeHandle(BasicType type) { return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE); } @@ -75,17 +76,18 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) bool complex = false; vespalib::string name = SelectUtils::extractFieldName(expr, complex); - AttributeGuard::UP ag(_amgr.getAttribute(name)); - if (ag->valid()) { + auto av = _amgr.readable_attribute_vector(name); + if (av) { if (complex) { ++_complexAttrs; // Don't try to optimize complex attribute references yet. _valueNode = expr.clone(); return; } - std::shared_ptr<search::AttributeVector> av(ag->getSP()); - if (av->getCollectionType() == CollectionType::SINGLE) { - if (isSingleValueThatWEHandle(av->getBasicType())) { + auto guard = av->makeReadGuard(false); + const auto* attr = guard->attribute(); + if (attr->getCollectionType() == CollectionType::SINGLE) { + if (isSingleValueThatWeHandle(attr->getBasicType())) { ++_svAttrs; auto it(_amap.find(name)); uint32_t idx(invalidIdx()); @@ -99,7 +101,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) idx = it->second; } assert(idx != invalidIdx()); - _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, av); + _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, idx); } else { ++_complexAttrs; // Don't try to optimize predicate/tensor/reference attributes yet. diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h index e9eb452889e..563eed75c32 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h @@ -15,6 +15,8 @@ namespace search { class IAttributeManager; } +namespace search::attribute { class ReadableAttributeVector; } + namespace proton { class SelectContext; @@ -44,7 +46,7 @@ public: const document::select::Node &selectNode() const; }; - using AttributeVectors = std::vector<std::shared_ptr<search::AttributeVector>>; + using AttributeVectors = std::vector<std::shared_ptr<search::attribute::ReadableAttributeVector>>; private: // Single value attributes referenced from selection expression diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index bddbfed371f..3fb32a9d1e0 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_map.h> #include <map> #include <vector> @@ -18,9 +17,8 @@ namespace proton { template <typename T> class HandlerMap { private: - typedef typename std::shared_ptr<T> HandlerSP; - typedef std::map<DocTypeName, HandlerSP> StdMap; - typedef typename StdMap::iterator MapIterator; + using HandlerSP = typename std::shared_ptr<T>; + using StdMap = std::map<DocTypeName, HandlerSP>; StdMap _handlers; @@ -40,8 +38,7 @@ public: size_t _offset; public: - typedef std::unique_ptr<Snapshot> UP; - + Snapshot() : _handlers(), _offset(0) { } Snapshot(const StdMap &map) : _handlers(), _offset(0) { _handlers.reserve(map.size()); for (auto itr : map) { @@ -49,6 +46,11 @@ public: } } Snapshot(std::vector<HandlerSP> &&handlers) : _handlers(std::move(handlers)), _offset(0) {} + Snapshot(Snapshot &&) noexcept = default; + Snapshot & operator = (Snapshot &&) noexcept = default; + Snapshot(const Snapshot &) = delete; + Snapshot & operator = (const Snapshot &) = delete; + bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } HandlerSP getSP() const { return _handlers[_offset]; } @@ -64,11 +66,7 @@ public: /** * Constructs a new instance of this class. */ - HandlerMap() - : _handlers() - { - // empty - } + HandlerMap() = default; /** * Registers a new handler for the given document type. If another handler @@ -83,7 +81,7 @@ public: putHandler(const DocTypeName &docTypeNameVer, const HandlerSP &handler) { - if (handler.get() == NULL) { + if ( ! handler) { throw vespalib::IllegalArgumentException(vespalib::make_string( "Handler is null for docType '%s'", docTypeNameVer.toString().c_str())); @@ -115,6 +113,20 @@ public: return HandlerSP(); } + /** + * Returns the handler for the given document type. If no handler was + * registered, this method returns a null pointer. + * + * @param docType The document type whose handler to return. + * @return The registered handler, if any. + */ + T * + getHandlerPtr(const DocTypeName &docTypeNameVer) const + { + const_iterator it = _handlers.find(docTypeNameVer); + return (it != _handlers.end()) ? it->second.get() : nullptr; + } + bool hasHandler(const HandlerSP &handler) const { bool found = false; for (const auto &kv : _handlers) { @@ -148,11 +160,7 @@ public: /** * Clear all handlers. */ - void - clear() - { - _handlers.clear(); - } + void clear() { _handlers.clear(); } /** * Create a snapshot of the handlers currently contained in this @@ -161,68 +169,15 @@ public: * * @return handler sequence **/ - std::unique_ptr<Snapshot> - snapshot() const - { - return std::unique_ptr<Snapshot>(new Snapshot(_handlers)); - } + Snapshot snapshot() const { return Snapshot(_handlers); } // we want to use snapshots rather than direct iteration to reduce locking; // the below functions should be deprecated when possible. - /** - * Returns a bidirectional iterator that points at the first element of the - * sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - iterator - begin() - { - return _handlers.begin(); - } - - /** - * Returns a const bidirectional iterator that points at the first element - * of the sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - const_iterator - begin() const - { - return _handlers.begin(); - } - - /** - * Returns a bidirectional iterator that points just beyond the end of the - * sequence. - * - * @return The end of this map. - */ - iterator - end() - { - return _handlers.end(); - } - - /** - * Returns a const bidirectional iterator that points just beyond the end of - * the sequence. - * - * @return The end of this map. - */ - const_iterator - end() const - { - return _handlers.end(); - } - - /** - * Returns the number of handlers in this map. - * - * @return the number of handlers. - */ + iterator begin() { return _handlers.begin(); } + const_iterator begin() const { return _handlers.begin(); } + iterator end() { return _handlers.end(); } + const_iterator end() const { return _handlers.end(); } size_t size() const { return _handlers.size(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp index 3275fbe06d4..ee713e19385 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp @@ -3,20 +3,22 @@ #include "selectcontext.h" #include "cachedselect.h" #include <vespa/document/select/value.h> -#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/readable_attribute_vector.h> +#include <cassert> namespace proton { using document::select::Value; using document::select::Context; -using search::AttributeGuard; using search::AttributeVector; +using search::attribute::AttributeReadGuard; namespace select { - struct Guards : public std::vector<AttributeGuard> { - using Parent = std::vector<AttributeGuard>; - using Parent::Parent; - }; +struct Guards : public std::vector<std::unique_ptr<AttributeReadGuard>> { + using Parent = std::vector<std::unique_ptr<AttributeReadGuard>>; + using Parent::Parent; +}; } SelectContext::SelectContext(const CachedSelect &cachedSelect) @@ -26,23 +28,30 @@ SelectContext::SelectContext(const CachedSelect &cachedSelect) _cachedSelect(cachedSelect) { } -SelectContext::~SelectContext() { } +SelectContext::~SelectContext() = default; void SelectContext::getAttributeGuards() { - _guards->resize(_cachedSelect.attributes().size()); - auto j(_cachedSelect.attributes().begin()); - for (std::vector<AttributeGuard>::iterator i(_guards->begin()), ie(_guards->end()); i != ie; ++i, ++j) { - *i = AttributeGuard(*j); + _guards->clear(); + _guards->reserve(_cachedSelect.attributes().size()); + for (const auto& attr : _cachedSelect.attributes()) { + _guards->emplace_back(attr->makeReadGuard(false)); } } - void SelectContext::dropAttributeGuards() { _guards->clear(); } +const search::attribute::IAttributeVector& +SelectContext::guarded_attribute_at_index(uint32_t index) const noexcept +{ + assert(index < _guards->size()); + assert((*_guards)[index].get() != nullptr); + return *((*_guards)[index])->attribute(); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h index d37c6eb6f14..4dd9f873088 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h @@ -3,6 +3,8 @@ #include <vespa/document/select/context.h> +namespace search::attribute { class IAttributeVector; } + namespace proton { class CachedSelect; @@ -19,6 +21,8 @@ public: void dropAttributeGuards(); uint32_t _docId; + + const search::attribute::IAttributeVector& guarded_attribute_at_index(uint32_t index) const noexcept; private: std::unique_ptr<select::Guards> _guards; const CachedSelect &_cachedSelect; diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp index 6cad54f134b..2c237074907 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp @@ -12,6 +12,7 @@ #include <vespa/document/select/invalidconstant.h> #include <vespa/document/select/valuenodes.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/iattributemanager.h> using document::select::And; @@ -97,10 +98,7 @@ SelectPruner::SelectPruner(const SelectPruner *rhs) } -SelectPruner::~SelectPruner() -{ -} - +SelectPruner::~SelectPruner() = default; void SelectPruner::visitAndBranch(const And &expr) @@ -395,7 +393,6 @@ SelectPruner::visitIdValueNode(const IdValueNode &expr) CloningVisitor::visitIdValueNode(expr); } - void SelectPruner::visitFieldValueNode(const FieldValueNode &expr) { @@ -406,32 +403,33 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) const document::DocumentType *docType = _repo.getDocumentType(_docType); bool complex = false; // Cannot handle attribute if complex expression vespalib::string name = SelectUtils::extractFieldName(expr, complex); - try { - std::unique_ptr<Field> fp(new Field(docType->getField(name))); - if (!fp) { + const bool is_imported = docType->has_imported_field_name(name); + if (complex || !is_imported) { + try { + std::unique_ptr<Field> fp(new Field(docType->getField(name))); + if (!fp) { + setInvalidVal(); + return; + } + } catch (FieldNotFoundException &) { + setInvalidVal(); + return; + } + try { + FieldPath path; + docType->buildFieldPath(path, expr.getFieldName()); + } catch (vespalib::IllegalArgumentException &) { + setInvalidVal(); + return; + } catch (FieldNotFoundException &) { setInvalidVal(); return; } - } catch (FieldNotFoundException &) { - setInvalidVal(); - return; - } - try { - FieldPath path; - docType->buildFieldPath(path, expr.getFieldName()); - } catch (vespalib::IllegalArgumentException &) { - setInvalidVal(); - return; - } catch (FieldNotFoundException &) { - setInvalidVal(); - return; } _constVal = false; if (!_hasFields) { // If we're working on removed document sub db then we have no fields. - _constVal = true; - _valueNode.reset(new NullValueNode()); - _priority = NullValPriority; + set_null_value_node(); return; } @@ -440,13 +438,18 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) bool svAttr = false; bool attrField = false; if (_amgr != nullptr) { - AttributeGuard::UP ag(_amgr->getAttribute(name)); - if (ag->valid()) { + auto attr = _amgr->readable_attribute_vector(name); + if (attr) { attrField = true; - auto av(ag->getSP()); - if (av->getCollectionType() == CollectionType::SINGLE && !complex) { + auto ag = attr->makeReadGuard(false); + if ((ag->attribute()->getCollectionType() == CollectionType::SINGLE) && !complex) { svAttr = true; } + } else if (is_imported) { + // Imported field present in document config but not yet in attribute config. + // Treat as missing (null) in document, as this matches behavior elsewhere in the pipeline. + set_null_value_node(); + return; } } if (!_hasDocuments && !svAttr) { @@ -538,6 +541,13 @@ SelectPruner::setInvalidConst() _node.reset(new InvalidConstant("invalid")); } +void +SelectPruner::set_null_value_node() +{ + _constVal = true; + _valueNode = std::make_unique<NullValueNode>(); + _priority = NullValPriority; +} void SelectPruner::setTernaryConst(bool val) diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h index 4350247d8b3..2c3729006c3 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h @@ -81,6 +81,7 @@ private: void setInvalidVal(); void setInvalidConst(); void setTernaryConst(bool val); + void set_null_value_node(); void resolveTernaryConst(bool wantInverted); bool isInvalidVal() const; bool isNullVal() const; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index 24ae86760c9..d2490985e77 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -94,7 +94,7 @@ LidStateVector::setBit(unsigned int idx) _highest = idx; } assert(!_bv.testBit(idx)); - _bv.slowSetBit(idx); + _bv.setBitAndMaintainCount(idx); ++_count; assert(_count == internalCount()); } @@ -105,7 +105,7 @@ LidStateVector::clearBit(unsigned int idx) { assert(idx < _bv.size()); assert(_bv.testBit(idx)); - _bv.slowClearBit(idx); + _bv.clearBitAndMaintainCount(idx); --_count; assert(_count == internalCount()); maybeUpdateLowest(); diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 480359f8382..b170f60d71f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -128,13 +128,13 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, if (searchHandler) { ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle); } else { - HandlerMap<ISearchHandler>::Snapshot::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - ISearchHandler::SP handler = snapshot->getSP(); + if (snapshot.valid()) { + ISearchHandler::SP handler = snapshot.getSP(); ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 6c97dc8c9ef..630ac66f4f1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -287,7 +287,9 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), estHits, reply->totalHitCount, request.ranking.c_str()); } - my_stats.queryCollateralTime(vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg()); + double querySetupTime = vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg(); + my_stats.queryCollateralTime(querySetupTime); // TODO: Remove in Vespa 8 + my_stats.querySetupTime(querySetupTime); { vespalib::duration duration = request.getTimeUsed(); std::lock_guard<std::mutex> guard(_statsLock); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index fff8c94c8be..8cc98816241 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -29,7 +29,8 @@ MatchingStats::MatchingStats() _softDoomed(0), _doomOvertime(), _softDoomFactor(INITIAL_SOFT_DOOM_FACTOR), - _queryCollateralTime(), + _queryCollateralTime(), // TODO: Remove in Vespa 8 + _querySetupTime(), _queryLatency(), _matchTime(), _groupingTime(), @@ -69,7 +70,8 @@ MatchingStats::add(const MatchingStats &rhs) _softDoomed += rhs.softDoomed(); _doomOvertime.add(rhs._doomOvertime); - _queryCollateralTime.add(rhs._queryCollateralTime); + _queryCollateralTime.add(rhs._queryCollateralTime); // TODO: Remove in Vespa 8 + _querySetupTime.add(rhs._querySetupTime); _queryLatency.add(rhs._queryLatency); _matchTime.add(rhs._matchTime); _groupingTime.add(rhs._groupingTime); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index f5eccdd1127..6b533aab7e5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -125,7 +125,8 @@ private: size_t _softDoomed; Avg _doomOvertime; double _softDoomFactor; - Avg _queryCollateralTime; + Avg _queryCollateralTime; // TODO: Remove in Vespa 8 + Avg _querySetupTime; Avg _queryLatency; Avg _matchTime; Avg _groupingTime; @@ -168,12 +169,19 @@ public: double softDoomFactor() const { return _softDoomFactor; } MatchingStats &updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::duration softLimit, vespalib::duration duration); + // TODO: Remove in Vespa 8 MatchingStats &queryCollateralTime(double time_s) { _queryCollateralTime.set(time_s); return *this; } double queryCollateralTimeAvg() const { return _queryCollateralTime.avg(); } size_t queryCollateralTimeCount() const { return _queryCollateralTime.count(); } double queryCollateralTimeMin() const { return _queryCollateralTime.min(); } double queryCollateralTimeMax() const { return _queryCollateralTime.max(); } + MatchingStats &querySetupTime(double time_s) { _querySetupTime.set(time_s); return *this; } + double querySetupTimeAvg() const { return _querySetupTime.avg(); } + size_t querySetupTimeCount() const { return _querySetupTime.count(); } + double querySetupTimeMin() const { return _querySetupTime.min(); } + double querySetupTimeMax() const { return _querySetupTime.max(); } + MatchingStats &queryLatency(double time_s) { _queryLatency.set(time_s); return *this; } double queryLatencyAvg() const { return _queryLatency.avg(); } size_t queryLatencyCount() const { return _queryLatency.count(); } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index f4b8203f8e2..05370920354 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -119,6 +119,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::update(const MatchingStats &stats) queries.inc(stats.queries()); queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); + querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), + stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), stats.queryLatencyMin(), stats.queryLatencyMax()); } @@ -131,6 +133,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent) queries("queries", {}, "Number of queries executed", this), softDoomedQueries("soft_doomed_queries", {}, "Number of queries hitting the soft timeout", this), queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), + querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { } @@ -152,6 +155,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics groupingTime("grouping_time", {}, "Average time (sec) spent on grouping", this), rerankTime("rerank_time", {}, "Average time (sec) spent on 2nd phase ranking", this), queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), + querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { softDoomFactor.set(MatchingStats::INITIAL_SOFT_DOOM_FACTOR); @@ -204,6 +208,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const Match stats.rerankTimeMin(), stats.rerankTimeMax()); queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); + querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), + stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), stats.queryLatencyMin(), stats.queryLatencyMax()); if (stats.getNumPartitions() > 0) { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 01ba271a08f..26dd52a8577 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -116,6 +116,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::LongCountMetric queries; metrics::LongCountMetric softDoomedQueries; metrics::DoubleAverageMetric queryCollateralTime; + metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; struct RankProfileMetrics : metrics::MetricSet { @@ -145,6 +146,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::DoubleAverageMetric groupingTime; metrics::DoubleAverageMetric rerankTime; metrics::DoubleAverageMetric queryCollateralTime; + metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; DocIdPartitions partitions; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index d34ec8d05a8..bf97bdda29a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -106,7 +106,7 @@ void doCleanAttributes(AttributeMetrics &attributes) { auto entries = attributes.release(); - for (const auto entry : entries) { + for (const auto &entry : entries) { attributes.parent()->unregisterMetric(*entry); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index e658bc0cfa0..8175e612bd4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_iterator.h" +#include <vespa/searchcore/proton/common/cachedselect.h> +#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/document/select/gid_filter.h> #include <vespa/document/select/node.h> #include <vespa/document/fieldvalue/document.h> @@ -31,7 +33,7 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss if (removed) { return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); } else { - ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->getSerializedSize(); + ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->serialize().size(); return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize); } } else { @@ -42,13 +44,6 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss } // namespace proton::<unnamed> bool -DocumentIterator::useDocumentSelection() const -{ - return (!_metaOnly && - !_selection.getDocumentSelection().getDocumentSelection().empty()); -} - -bool DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const { if (!meta.valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h index 285b3cb2afa..67242e8220f 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h @@ -3,8 +3,6 @@ #pragma once #include "i_document_retriever.h" -#include <vespa/searchcore/proton/common/cachedselect.h> -#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> @@ -32,10 +30,7 @@ private: storage::spi::IterateResult::List _list; - bool useDocumentSelection() const; bool checkMeta(const search::DocumentMetaData &meta) const; - bool checkDoc(const document::Document &doc) const; - bool checkDoc(const SelectContext &sc) const; void fetchCompleteSource(const IDocumentRetriever & source, storage::spi::IterateResult::List & list); bool isWeakRead() const { return _readConsistency == ReadConsistency::WEAK; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index 06969167ab3..5b62a290353 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -20,15 +20,15 @@ PersistenceHandlerMap::putHandler(document::BucketSpace bucketSpace, return _map[bucketSpace].putHandler(docType, handler); } -IPersistenceHandler::SP +IPersistenceHandler * PersistenceHandlerMap::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return itr->second.getHandler(docType); + return itr->second.getHandlerPtr(docType); } - return IPersistenceHandler::SP(); + return nullptr; } IPersistenceHandler::SP @@ -42,7 +42,7 @@ PersistenceHandlerMap::removeHandler(document::BucketSpace bucketSpace, return IPersistenceHandler::SP(); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot() const { std::vector<IPersistenceHandler::SP> handlers; @@ -52,47 +52,17 @@ PersistenceHandlerMap::getHandlerSnapshot() const } } size_t handlersSize = handlers.size(); - return std::make_unique<HandlerSnapshot> - (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)), - handlersSize); -} - -namespace { - -struct EmptySequence : public vespalib::Sequence<IPersistenceHandler *> { - virtual bool valid() const override { return false; } - virtual IPersistenceHandler *get() const override { return nullptr; } - virtual void next() override { } - static EmptySequence::UP make() { return std::make_unique<EmptySequence>(); } -}; - + return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers)), handlersSize); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return std::make_unique<HandlerSnapshot>(itr->second.snapshot(), itr->second.size()); + return HandlerSnapshot(itr->second.snapshot(), itr->second.size()); } - return std::make_unique<HandlerSnapshot>(EmptySequence::make(), 0); -} - -namespace { - -class SequenceOfOne : public vespalib::Sequence<IPersistenceHandler *> { -private: - bool _done; - IPersistenceHandler *_value; -public: - SequenceOfOne(IPersistenceHandler *value) : _done(false), _value(value) {} - - virtual bool valid() const override { return !_done; } - virtual IPersistenceHandler *get() const override { return _value; } - virtual void next() override { _done = true; } - static SequenceOfOne::UP make(IPersistenceHandler *value) { return std::make_unique<SequenceOfOne>(value); } -}; - + return HandlerSnapshot(); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h index 003c378d6a7..64241e1ad2b 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -20,16 +20,17 @@ class IPersistenceHandler; */ class PersistenceHandlerMap { public: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + using PersistenceHandlerSequence = DocTypeToHandlerMap::Snapshot; using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>; class HandlerSnapshot { private: - PersistenceHandlerSequence::UP _handlers; - size_t _size; + PersistenceHandlerSequence _handlers; + size_t _size; public: - using UP = std::unique_ptr<HandlerSnapshot>; - HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_) + HandlerSnapshot() : _handlers(), _size(0) {} + HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_, size_t size_) : _handlers(std::move(handlers_)), _size(size_) {} @@ -37,12 +38,12 @@ public: HandlerSnapshot & operator = (const HandlerSnapshot &) = delete; size_t size() const { return _size; } - PersistenceHandlerSequence &handlers() { return *_handlers; } - static PersistenceHandlerSequence::UP release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } + PersistenceHandlerSequence &handlers() { return _handlers; } + static PersistenceHandlerSequence release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } }; private: - using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + struct BucketSpaceHash { std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } @@ -53,15 +54,11 @@ private: public: PersistenceHandlerMap(); - PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType, - const PersistenceHandlerSP &handler); - PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType); - PersistenceHandlerSP getHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot() const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const PersistenceHandlerSP &handler); + PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); + IPersistenceHandler * getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot() const; + HandlerSnapshot getHandlerSnapshot(document::BucketSpace bucketSpace) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 1f862b07048..2ede5d45f7e 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -40,7 +40,7 @@ protected: std::mutex _lock; vespalib::CountDownLatch _latch; public: - ResultHandlerBase(uint32_t waitCnt); + explicit ResultHandlerBase(uint32_t waitCnt); ~ResultHandlerBase(); void await() { _latch.await(); } }; @@ -55,11 +55,11 @@ class GenericResultHandler : public ResultHandlerBase, public IGenericResultHand private: Result _result; public: - GenericResultHandler(uint32_t waitCnt) : + explicit GenericResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), _result() { } - ~GenericResultHandler(); + ~GenericResultHandler() override; void handle(const Result &result) override { if (result.hasError()) { std::lock_guard<std::mutex> guard(_lock); @@ -109,7 +109,7 @@ class SynchronizedBucketIdListResultHandler : public ResultHandlerBase, public BucketIdListResultHandler { public: - SynchronizedBucketIdListResultHandler(uint32_t waitCnt) + explicit SynchronizedBucketIdListResultHandler(uint32_t waitCnt) : ResultHandlerBase(waitCnt), BucketIdListResultHandler() { } @@ -163,17 +163,21 @@ BucketInfoResultHandler::~BucketInfoResultHandler() = default; } -PersistenceEngine::HandlerSnapshot::UP -PersistenceEngine::getHandlerSnapshot() const +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const WriteGuard &) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(); } -PersistenceEngine::HandlerSnapshot::UP -PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace) const +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const +{ + return _handlers.getHandlerSnapshot(bucketSpace); +} + +PersistenceEngine::HandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const WriteGuard &, document::BucketSpace bucketSpace) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandlerSnapshot(bucketSpace); } @@ -202,27 +206,23 @@ PersistenceEngine::~PersistenceEngine() IPersistenceHandler::SP -PersistenceEngine::putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, - const IPersistenceHandler::SP &handler) +PersistenceEngine::putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType,const IPersistenceHandler::SP &handler) { - std::lock_guard<std::mutex> guard(_lock); return _handlers.putHandler(bucketSpace, docType, handler); } -IPersistenceHandler::SP -PersistenceEngine::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const +IPersistenceHandler * +PersistenceEngine::getHandler(const ReadGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) const { - std::lock_guard<std::mutex> guard(_lock); return _handlers.getHandler(bucketSpace, docType); } IPersistenceHandler::SP -PersistenceEngine::removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) +PersistenceEngine::removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) { // TODO: Grab bucket list and treat them as modified - std::lock_guard<std::mutex> guard(_lock); return _handlers.removeHandler(bucketSpace, docType); } @@ -232,9 +232,9 @@ PersistenceEngine::initialize() { std::unique_lock<std::shared_timed_mutex> wguard(getWLock()); LOG(debug, "Begin initializing persistence handlers"); - HandlerSnapshot::UP snap = getHandlerSnapshot(); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(wguard); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->initialize(); } LOG(debug, "Done initializing persistence handlers"); @@ -260,10 +260,10 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const BucketIdListResult::List emptyList; return BucketIdListResult(emptyList); } - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); BucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListBuckets(resultHandler); } return resultHandler.getResult(); @@ -275,10 +275,10 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); saveClusterState(bucketSpace, calc); - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetClusterState(calc, resultHandler); } resultHandler.await(); @@ -292,10 +292,10 @@ PersistenceEngine::setActiveState(const Bucket& bucket, storage::spi::BucketInfo::ActiveState newState) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(bucket.getBucketSpace()); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetActiveState(bucket, newState, resultHandler); } resultHandler.await(); @@ -309,10 +309,10 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const // Runs in SPI thread. // No handover to write threads in persistence handlers. std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); BucketInfoResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetBucketInfo(b, resultHandler); } return resultHandler.getResult(); @@ -338,7 +338,7 @@ PersistenceEngine::put(const Bucket& b, Timestamp t, const document::Document::S return Result(Result::ErrorType::PERMANENT_ERROR, make_string("Old id scheme not supported in elastic mode (%s)", doc->getId().toString().c_str())); } - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return Result(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -360,7 +360,7 @@ PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, C make_string("Old id scheme not supported in elastic mode (%s)", did.toString().c_str())); } DocTypeName docType(did.getDocType()); - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return RemoveResult(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -414,7 +414,7 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP return UpdateResult(Result::ErrorType::PERMANENT_ERROR, make_string("Update operation rejected due to bad id (%s, %s)", upd->getId().toString().c_str(), docType.getName().c_str())); } - IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (handler) { TransportLatch latch(1); @@ -432,9 +432,9 @@ PersistenceEngine::GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(b.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { @@ -462,19 +462,19 @@ PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet IncludedVersions versions, Context & context) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(bucket.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection, versions, _defaultSerializedSize, _ignoreMaxBytes); - entry->bucket_guards.reserve(snapshot->size()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + entry->bucket_guards.reserve(snapshot.size()); + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket)); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { entry->it.add((*retrievers)[i]); } } - entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot)); + entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); std::lock_guard<std::mutex> guard(_iterators_lock); static IteratorId id_counter(0); @@ -541,10 +541,10 @@ PersistenceEngine::createBucket(const Bucket &b, Context &) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "createBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleCreateBucket(feedtoken::make(latch), b); } latch.await(); @@ -557,10 +557,10 @@ PersistenceEngine::deleteBucket(const Bucket& b, Context&) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleDeleteBucket(feedtoken::make(latch), b); } latch.await(); @@ -578,10 +578,10 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const std::lock_guard<std::mutex> guard(_lock); extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); - SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + SynchronizedBucketIdListResultHandler resultHandler(snap.size() + extraModifiedBuckets.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetModifiedBuckets(resultHandler); } for (const auto & item : extraModifiedBuckets) { @@ -599,10 +599,10 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); assert(source.getBucketSpace() == target1.getBucketSpace()); assert(source.getBucketSpace() == target2.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(source.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, source.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSplit(feedtoken::make(latch), source, target1, target2); } latch.await(); @@ -617,10 +617,10 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); assert(source1.getBucketSpace() == target.getBucketSpace()); assert(source2.getBucketSpace() == target.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(target.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, target.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleJoin(feedtoken::make(latch), source1, source2, target); } latch.await(); @@ -722,19 +722,18 @@ public: }; void -PersistenceEngine::populateInitialBucketDB(BucketSpace bucketSpace, - IPersistenceHandler &targetHandler) +PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler) { - HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(guard, bucketSpace); - size_t snapSize(snap->size()); + size_t snapSize(snap.size()); size_t flawed = 0; // handleListActiveBuckets() runs in SPI thread. // No handover to write threads in persistence handlers. ActiveBucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListActiveBuckets(resultHandler); } typedef std::map<document::BucketId, size_t> BucketIdMap; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a6c696d08fb..5d3be07c532 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -5,12 +5,9 @@ #include "i_resource_write_filter.h" #include "persistence_handler_map.h" #include "ipersistencehandler.h" -#include <vespa/document/bucket/bucketspace.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> -#include <vespa/searchcore/proton/common/handlermap.hpp> #include <mutex> #include <shared_mutex> -#include <unordered_map> namespace proton { @@ -18,7 +15,7 @@ class IPersistenceEngineOwner; class PersistenceEngine : public storage::spi::AbstractPersistenceProvider { private: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using PersistenceHandlerSequence = PersistenceHandlerMap::PersistenceHandlerSequence; using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; using DocumentUpdate = document::DocumentUpdate; using Bucket = storage::spi::Bucket; @@ -43,7 +40,7 @@ private: using UpdateResult = storage::spi::UpdateResult; struct IteratorEntry { - PersistenceHandlerSequence::UP handler_sequence; + PersistenceHandlerSequence handler_sequence; DocumentIterator it; bool in_use; std::vector<BucketGuard::UP> bucket_guards; @@ -75,9 +72,13 @@ private: mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_timed_mutex _rwMutex; - IPersistenceHandler::SP getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot() const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + using ReadGuard = std::shared_lock<std::shared_timed_mutex>; + using WriteGuard = std::unique_lock<std::shared_timed_mutex>; + + IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const; + HandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const; void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; @@ -89,9 +90,8 @@ public: ssize_t defaultSerializedSize, bool ignoreMaxBytes); ~PersistenceEngine() override; - IPersistenceHandler::SP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, - const IPersistenceHandler::SP &handler); - IPersistenceHandler::SP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); + IPersistenceHandler::SP putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler); + IPersistenceHandler::SP removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType); // Implements PersistenceProvider Result initialize() override; @@ -121,8 +121,8 @@ public: void destroyIterators(); void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); - void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler); - std::unique_lock<std::shared_timed_mutex> getWLock() const; + void populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler); + WriteGuard getWLock() const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index 1d3b2165c80..57f1ee74e60 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -116,7 +116,7 @@ BucketHandler::handleGetBucketInfo(const Bucket &bucket, // Called by SPI thread. // BucketDBOwner ensures synchronization between SPI thread and // master write thread in document database. - BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGet(bucket); + BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGetBucketInfo(bucket); LOG(spam, "handleGetBucketInfo(%s): %s", bucket.toString().c_str(), bucketInfo.toString().c_str()); resultHandler.handle(BucketInfoResult(bucketInfo)); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 9445a0a5206..1a6fc6cdbfd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -163,7 +163,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) _listeners() { } -DiskMemUsageFilter::~DiskMemUsageFilter() { } +DiskMemUsageFilter::~DiskMemUsageFilter() = default; void DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in) diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index 40c74808e72..b05cc261728 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -15,11 +15,7 @@ class DiskMemUsageState ResourceUsageState _memoryState; public: - DiskMemUsageState() - : _diskState(), - _memoryState() - { - } + DiskMemUsageState() = default; DiskMemUsageState(const ResourceUsageState &diskState_, const ResourceUsageState &memoryState_) : _diskState(diskState_), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index 360a0b12111..abfc840fc7e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentretriever.h" +#include <vespa/document/datatype/arraydatatype.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/document_field_retriever.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/searchlib/attribute/attributevector.h> @@ -28,6 +31,18 @@ using vespalib::geo::ZCurve; namespace proton { +namespace { + +bool is_array_of_position_type(const document::DataType& field_type) noexcept { + const auto* arr_type = dynamic_cast<const document::ArrayDataType*>(&field_type); + if (!arr_type) { + return false; + } + return (arr_type->getNestedType() == PositionDataType::getInstance()); +} + +} + DocumentRetriever ::DocumentRetriever(const DocTypeName &docTypeName, const DocumentTypeRepo &repo, @@ -47,7 +62,7 @@ DocumentRetriever int32_t positionDataTypeId = PositionDataType::getInstance().getId(); LOG(debug, "checking document type '%s' for position fields", docTypeName.getName().c_str()); for (const document::Field * field : fields) { - if (field->getDataType().getId() == positionDataTypeId) { + if ((field->getDataType().getId() == positionDataTypeId) || is_array_of_position_type(field->getDataType())) { LOG(debug, "Field '%s' is a position field", field->getName().data()); const vespalib::string & zcurve_name = PositionDataType::getZCurveFieldName(field->getName()); AttributeGuard::UP attr = attr_manager.getAttribute(zcurve_name); @@ -72,19 +87,40 @@ FieldValue::UP positionFromZcurve(int64_t zcurve) { ZCurve::decode(zcurve, &x, &y); FieldValue::UP value = PositionDataType::getInstance().createFieldValue(); - StructFieldValue *position = static_cast<StructFieldValue *>(value.get()); + auto *position = static_cast<StructFieldValue *>(value.get()); position->set(PositionDataType::FIELD_X, x); position->set(PositionDataType::FIELD_Y, y); return value; } +std::unique_ptr<document::FieldValue> +zcurve_array_attribute_to_field_value(const document::Field& field, + const search::attribute::IAttributeVector& attr, + DocumentIdT lid) +{ + search::attribute::AttributeContent<int64_t> zc_elems; + zc_elems.fill(attr, lid); + auto new_fv = field.createValue(); + auto& new_array_fv = dynamic_cast<document::ArrayFieldValue&>(*new_fv); + new_array_fv.reserve(zc_elems.size()); + for (int64_t zc : zc_elems) { + new_array_fv.append(positionFromZcurve(zc)); + } + return new_fv; +} + void fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::PositionFields & possiblePositionFields, const IAttributeManager & attr_manager) { for (const auto & it : possiblePositionFields) { - AttributeGuard::UP attr = attr_manager.getAttribute(it.second); - if (!(*attr)->isUndefined(lid)) { - int64_t zcurve = (*attr)->getInt(lid); - doc.setValue(*it.first, *positionFromZcurve(zcurve)); + auto attr_guard = attr_manager.getAttribute(it.second); + auto& attr = *attr_guard; + if (!attr->isUndefined(lid)) { + if (attr->hasArrayType()) { + doc.setFieldValue(*it.first, zcurve_array_attribute_to_field_value(*it.first, *attr, lid)); + } else { + int64_t zcurve = attr->getInt(lid); + doc.setValue(*it.first, *positionFromZcurve(zcurve)); + } } else { doc.remove(*it.first); // Don't resurrect old values from the docstore. } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index 5a6a990df9b..faee5914a97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -49,7 +49,7 @@ handleProgress(TlsReplayProgress &progress, SerialNum currentSerial) void handlePacket(PacketWrapper::SP wrap, EntryHandler entryHandler) { - vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().c_str(), wrap->packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().data(), wrap->packet.getHandle().size()); while (handle.size() > 0) { Packet::Entry entry; entry.deserialize(handle); diff --git a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h index c70a6c502f1..63ff269580e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h @@ -36,7 +36,7 @@ public: */ virtual void unBlock(BlockedReason reason) = 0; - virtual IBlockableMaintenanceJob *asBlockable() override { return this; } + IBlockableMaintenanceJob *asBlockable() override { return this; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h index 6d0739e1aed..df6889ecf04 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h @@ -31,7 +31,7 @@ public: _interval(interval) {} - virtual ~IMaintenanceJob() {} + virtual ~IMaintenanceJob() = default; virtual const vespalib::string &getName() const { return _name; } virtual vespalib::duration getDelay() const { return _delay; } diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp index 6c0c0863fe1..e535b05393c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp @@ -41,9 +41,7 @@ MoveOperationLimiter::MoveOperationLimiter(IBlockableMaintenanceJob *job, { } -MoveOperationLimiter::~MoveOperationLimiter() -{ -} +MoveOperationLimiter::~MoveOperationLimiter() = default; void MoveOperationLimiter::clearJob() diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 4daf3e895af..28de4dff917 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -594,10 +594,10 @@ Proton::addDocumentDB(const document::DocumentType &docType, auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret); if (!_isInitializing) { _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler); - _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler); + _persistenceEngine->populateInitialBucketDB(persistenceWGuard, bucketSpace, *persistenceHandler); } // TODO: Fix race with new cluster state setting. - _persistenceEngine->putHandler(bucketSpace, docTypeName, persistenceHandler); + _persistenceEngine->putHandler(persistenceWGuard, bucketSpace, docTypeName, persistenceHandler); } auto searchHandler = std::make_shared<SearchHandlerProxy>(ret); _summaryEngine->putSearchHandler(docTypeName, searchHandler); @@ -629,7 +629,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) // Not allowed to get to service layer to call pause(). std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock()); IPersistenceHandler::SP oldHandler; - oldHandler = _persistenceEngine->removeHandler(old->getBucketSpace(), docTypeName); + oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName); if (_initComplete && oldHandler) { // TODO: Fix race with bucket db modifying ops. _persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler); diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp index bfc59dee35e..bbd02d7efce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp @@ -14,7 +14,7 @@ namespace proton { void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf, DoneCallback onDone) { - Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.c_str(), buf.size())); + Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.data(), buf.size())); Packet packet; packet.add(entry); packet.close(); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index e154c6761e2..692b05899f4 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -128,13 +128,13 @@ SummaryEngine::getDocsums(DocsumRequest::UP req) if (searchHandler) { reply = searchHandler->getDocsums(*req); } else { - vespalib::Sequence<ISearchHandler*>::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - reply = snapshot->get()->getDocsums(*req); // use the first handler + if (snapshot.valid()) { + reply = snapshot.get()->getDocsums(*req); // use the first handler } } updateDocsumMetrics(vespalib::to_s(req->getTimeUsed()), getNumDocs(*reply)); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 87fa2053508..57c86855217 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -78,6 +78,9 @@ public: void asyncForAttribute(const vespalib::string & name, std::unique_ptr<IAttributeFunctor> func) const override { _mock.asyncForAttribute(name, std::move(func)); } + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override { + return _mock.readable_attribute_vector(name); + } }; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 2c8eff4f4ad..d754fd78394 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -210,6 +210,7 @@ vespa_define_module( src/tests/sortspec src/tests/stringenum src/tests/tensor/dense_tensor_store + src/tests/tensor/hnsw_index src/tests/transactionlog src/tests/transactionlogstress src/tests/true diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 620a139d451..4f8a4ad7345 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/app.h> #include <unistd.h> +#include <random> #include <vespa/log/log.h> LOG_SETUP("documentstore.benchmark"); @@ -65,17 +66,13 @@ BenchmarkDataStoreApp::Main() void BenchmarkDataStoreApp::read(size_t numReads, size_t perChunk, const IDataStore * dataStore) { vespalib::DataBuffer buf; - struct random_data rstate; - char state[8]; - memset(state, 0, sizeof(state)); - memset(&rstate, 0, sizeof(rstate)); + std::minstd_rand rng; const size_t docIdLimit(dataStore->getDocIdLimit()); assert(docIdLimit > 0); - initstate_r(getpid(), state, sizeof(state), &rstate); - assert(srandom_r(getpid(), &rstate) == 0); + rng.seed(getpid()); int32_t rnd(0); for ( size_t i(0); i < numReads; i++) { - random_r(&rstate, &rnd); + rnd = rng(); uint32_t lid(rnd%docIdLimit); for (uint32_t j(lid); j < std::min(docIdLimit, lid+perChunk); j++) { dataStore->read(j, buf); diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 5990b3ec805..46aca14325f 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -78,7 +78,7 @@ generate(uint64_t serialNum, size_t chunks, FastOS_FileInterface & idxFile, size fprintf(stdout, "Failed with lengthError %ld due to '%s'\n", lengthError, e.what()); } } - idxFile.Write2(os.c_str(), os.size()); + idxFile.Write2(os.data(), os.size()); return serialNum; } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 6a87e0c6d46..807eb3aa7ce 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -386,6 +386,56 @@ public class EvaluationTestCase { // tensor result dimensions are given from argument dimensions, not the resulting values tester.assertEvaluates("tensor(x{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{}):{ {x:1}:1 }"); tester.assertEvaluates("tensor(x{},y{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{},y{}):{ {x:1,y:0}:1, {x:2,y:1}:1 }"); + + } + + @Test + public void testTake() { + EvaluationTester tester = new EvaluationTester(); + + // numpy.take(a, indices, axis) with tensors. + + // 1 dim input, 1 dim indices + tester.assertEvaluates("tensor(d0[3]):[1, 3, 5]", + "tensor(d0[3])(tensor0{a0:(tensor1{indices0:(d0)})})", + "tensor(a0[6]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[3]):[0, 2, 4]"); + + // 1 dim input, 1 dim indices - negative indices + tester.assertEvaluates("tensor(d0[3]):[1, 5, 3]", + "tensor(d0[3])(tensor0{a0:(fmod(6 + tensor1{indices0:(d0)}, 6) ) })", + "tensor(a0[6]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[3]):[0, -2, -4]"); + + // 2 dim input, 1 dim indices - axis 0 + tester.assertEvaluates("tensor(d0[4],d1[2]):[5, 6, 3, 4, 1, 2, 5, 6]", + "tensor(d0[4],d1[2])(tensor0{a0:(tensor1{indices0:(d0)}),a1:(d1)})", + "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[4]):[2, 1, 0, 2]"); + + // 1 dim input, 2 dim indices - axis 0 + tester.assertEvaluates("tensor(d0[2],d1[2]):[1, 2, 4, 6]", + "tensor(d0[2],d1[2])(tensor0{a0:(tensor1{indices0:(d0),indices1:(d1)}) })", + "tensor(a0[6]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[2],indices1[2]):[0, 1, 3, 5]"); + + // 2 dim input, 2 dim indices - axis 0 + tester.assertEvaluates("tensor(d0[2],d1[2],d2[2]):[1,2,3,4,3,4,5,6]", + "tensor(d0[2],d1[2],d2[2])(tensor0{a0:(tensor1{indices0:(d0),indices1:(d1)}),a1:(d2)})", + "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[2],indices1[2]):[0, 1, 1, 2]"); + + // 2 dim input, 1 dim indices - axis 1 + tester.assertEvaluates("tensor(d0[3],d1[4]):[1,2,1,2,3,4,3,4,5,6,5,6]", + "tensor(d0[3],d1[4])(tensor0{a0:(d0), a1:(tensor1{indices0:(d1)}) })", + "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]", + "tensor(indices0[4]):[0, 1, 0, 1]"); + + // 2 dim input, 2 dim indices - axis 1 + tester.assertEvaluates("tensor(d0[3],d1[1],d2[2]):[1,3,4,6,7,9]", + "tensor(d0[3],d1[1],d2[2])(tensor0{a0:(d0), a1:(tensor1{indices0:(d1),indices1:(d2)}) })", // can add an if + "tensor(a0[3],a1[3]):[1, 2, 3, 4, 5, 6, 7, 8, 9]", + "tensor(indices0[1],indices1[2]):[0, 2]"); } @Test diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 513e94321e1..41465f991e3 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -489,9 +489,9 @@ TEST("testResultNodes") { double d(786324.78); nbostream os; os << j << d; - RawResultNode r1(os.c_str(), sizeof(j)); + RawResultNode r1(os.data(), sizeof(j)); EXPECT_EQUAL(r1.getInteger(), 789); - RawResultNode r2(os.c_str() + sizeof(j), sizeof(d)); + RawResultNode r2(os.data() + sizeof(j), sizeof(d)); EXPECT_EQUAL(r2.getFloat(), 786324.78); StringResultNode s1, s2("a"), s3("a"), s4("b"), s5("bb"); @@ -560,7 +560,7 @@ void testStreaming(const Identifiable &v) { EXPECT_EQUAL(os2.size(), os3.size()); ASSERT_TRUE(os2.size() == os3.size()); - EXPECT_EQUAL(0, memcmp(os2.c_str(), os3.c_str(), os3.size())); + EXPECT_EQUAL(0, memcmp(os2.data(), os3.data(), os3.size())); } TEST("testTimeStamp") { @@ -1209,7 +1209,7 @@ TEST("testArithmeticOperations") { testAdd(createScalarInt(I1), createScalarInt(I2), 3469774562ull, 3469774562ull); testAdd(createScalarInt(I1), createScalarFloat(F2), 1793253251ull, 1793253250.767681239); testAdd(createScalarFloat(F1), createScalarFloat(F2), 11, 10.878668839 ); - testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, 3006427292488851361ull); + testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, static_cast<double>(3006427292488851361ull)); testMultiply(createScalarInt(I1), createScalarFloat(F2), 17515926039ull, 1793253241.0*9.767681239); testMultiply(createScalarFloat(F1), createScalarFloat(F2), 11, 10.8517727372816364 ); diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 62cde4d2c9c..7d09b2aa0b8 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -34,6 +34,7 @@ private: void testGuards(); void testConfigConvert(); void testContext(); + void can_get_readable_attribute_vector_by_name(); bool assertDataType(BT::Type exp, @@ -377,6 +378,21 @@ AttributeManagerTest::testContext() } } +void +AttributeManagerTest::can_get_readable_attribute_vector_by_name() +{ + auto attr = AttributeFactory::createAttribute("cool_attr", Config(BT::INT32, CT::SINGLE)); + // Ensure there's something to actually load, or fetching the attribute will throw. + attr->addDocs(64); + attr->commit(); + AttributeManager manager; + manager.add(attr); + auto av = manager.readable_attribute_vector("cool_attr"); + EXPECT_EQUAL(av.get(), static_cast<ReadableAttributeVector*>(attr.get())); + av = manager.readable_attribute_vector("uncool_attr"); + EXPECT_TRUE(av.get() == nullptr); +} + int AttributeManagerTest::Main() { TEST_INIT("attributemanager_test"); @@ -385,6 +401,7 @@ int AttributeManagerTest::Main() testGuards(); testConfigConvert(); testContext(); + can_get_readable_attribute_vector_by_name(); TEST_DONE(); } diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index cfa05b7a765..48a06bec9e8 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -219,6 +219,15 @@ TEST_F("Weighted floating point attribute values can be retrieved via reference" assert_multi_value_matches<WeightedFloat>(f, DocId(3), doc7_values); } +TEST_F("isUndefined() works for primitive attribute type", Fixture) { + reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>( + f, BasicType::INT32, + {{DocId(3), dummy_gid(7), DocId(7), 5678}}); + + EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(3))); // Mapped + EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(2))); // Not mapped +} + struct SingleStringAttrFixture : Fixture { SingleStringAttrFixture() : Fixture() { setup(); @@ -255,6 +264,11 @@ TEST_F("findEnum() returns target vector enum via reference", SingleStringAttrFi EXPECT_EQUAL(expected_handle, actual_handle); } +TEST_F("isUndefined() works for enumerated attribute type", SingleStringAttrFixture) { + EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(2))); // Mapped + EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(3))); // Not mapped +} + // Note: assumes that fixture has set up a string enum of value "foo" in target attribute template <typename FixtureType> void verify_get_string_from_enum_is_mapped(FixtureType& f) { diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 4818287b429..2eafeab20bd 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -123,6 +123,15 @@ public: return IAttributeContext::UP(); } + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override { + if (name == field) { + return _attribute_vector; + } else if (name == other) { + return _other; + } + return {}; + } + void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; }; diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index e9addae07b9..24be21f65ec 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -100,6 +100,10 @@ public: void asyncForAttribute(const vespalib::string &, std::unique_ptr<IAttributeFunctor>) const override { assert(!"Not implemented"); } + + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override { + return _attribute_vector; + } }; constexpr uint32_t DOCID_LIMIT = 3; diff --git a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt index a705fd5ecb5..377d91bf634 100644 --- a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_searchcontext_test_app TEST SOURCES - searchcontext.cpp + searchcontext_test.cpp DEPENDS searchlib searchlib_test diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 7d4a2d63355..416ddb5fbc0 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -1,24 +1,27 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeiterators.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/elementiterator.h> #include <vespa/searchlib/attribute/flagattribute.h> +#include <vespa/searchlib/attribute/multistringattribute.h> +#include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> -#include <vespa/searchlib/attribute/multistringattribute.h> -#include <vespa/searchlib/attribute/elementiterator.h> #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include <vespa/searchlib/queryeval/hitcollector.h> +#include <vespa/searchlib/parsequery/parse.h> +#include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/executeinfo.h> +#include <vespa/searchlib/queryeval/hitcollector.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/compress.h> -#include <vespa/searchlib/test/searchiteratorverifier.h> -#include <vespa/searchlib/query/query_term_simple.h> -#include <vespa/searchlib/parsequery/parse.h> -#include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/log/log.h> LOG_SETUP("searchcontext_test"); @@ -43,22 +46,24 @@ isUnsignedSmallIntAttribute(const AttributeVector &a) } -typedef AttributeVector::SP AttributePtr; -typedef std::unique_ptr<AttributeVector::SearchContext> SearchContextPtr; -typedef AttributeVector::SearchContext SearchContext; -using attribute::Config; +using AttributePtr = AttributeVector::SP; +using ResultSetPtr = std::unique_ptr<ResultSet>; +using SearchBasePtr = queryeval::SearchIterator::UP; +using SearchContext = AttributeVector::SearchContext; +using SearchContextPtr = std::unique_ptr<AttributeVector::SearchContext>; +using largeint_t = AttributeVector::largeint_t; + using attribute::BasicType; using attribute::CollectionType; -typedef AttributeVector::largeint_t largeint_t; -typedef queryeval::SearchIterator::UP SearchBasePtr; -typedef std::unique_ptr<ResultSet> ResultSetPtr; - -using queryeval::HitCollector; -using queryeval::SearchIterator; +using attribute::Config; +using attribute::SearchContextParams; using fef::MatchData; using fef::TermFieldMatchData; using fef::TermFieldMatchDataArray; using fef::TermFieldMatchDataPosition; +using queryeval::HitCollector; +using queryeval::SearchIterator; +using queryeval::SimpleResult; class DocSet : public std::set<uint32_t> { @@ -267,6 +272,9 @@ private: void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, const Config &cfg, int64_t maxValue); void requireThatOutOfBoundsSearchTermGivesZeroHits(); + void single_bool_attribute_search_context_handles_true_and_false_queries(); + void single_bool_attribute_search_iterator_handles_true_and_false_queries(); + // init maps with config objects void initIntegerConfig(); void initFloatConfig(); @@ -1825,6 +1833,87 @@ SearchContextTest::requireThatOutOfBoundsSearchTermGivesZeroHits() } } +class BoolAttributeFixture { +private: + search::SingleBoolAttribute _attr; + +public: + BoolAttributeFixture(const SimpleResult& true_docs, uint32_t num_docs) + : _attr("bool_attr", search::GrowStrategy()) + { + _attr.addDocs(num_docs); + for (uint32_t i = 0; i < true_docs.getHitCount(); ++i) { + uint32_t docid = true_docs.getHit(i); + _attr.update(docid, 1); + } + _attr.commit(); + } + search::AttributeVector::SearchContext::UP create_search_context(const std::string& term) const { + return _attr.getSearch(std::make_unique<search::QueryTermSimple>(term, search::QueryTermSimple::WORD), + SearchContextParams().useBitVector(true)); + } + SimpleResult search_context(const std::string& term) const { + auto search_ctx = create_search_context(term); + SimpleResult result; + int32_t weight = 10; + for (uint32_t docid = 1; docid < _attr.getNumDocs(); ++docid) { + bool match_1 = search_ctx->matches(docid); + bool match_2 = search_ctx->matches(docid, weight); + EXPECT_EQUAL(match_1, match_2); + EXPECT_EQUAL(match_2 ? 1 : 0, weight); + if (match_1) { + result.addHit(docid); + } + weight = 10; + } + return result; + } + SimpleResult search_iterator(const std::string& term, bool strict) const { + auto search_ctx = create_search_context(term); + TermFieldMatchData tfmd; + auto itr = search_ctx->createIterator(&tfmd, strict); + SimpleResult result; + if (strict) { + result.searchStrict(*itr, _attr.getNumDocs()); + } else { + result.search(*itr, _attr.getNumDocs()); + } + return result; + } +}; + +void +SearchContextTest::single_bool_attribute_search_context_handles_true_and_false_queries() +{ + BoolAttributeFixture f(SimpleResult().addHit(3).addHit(5).addHit(7), 9); + + auto true_exp = SimpleResult().addHit(3).addHit(5).addHit(7); + EXPECT_EQUAL(true_exp, f.search_context("true")); + EXPECT_EQUAL(true_exp, f.search_context("1")); + + auto false_exp = SimpleResult().addHit(1).addHit(2).addHit(4).addHit(6).addHit(8); + EXPECT_EQUAL(false_exp, f.search_context("false")); + EXPECT_EQUAL(false_exp, f.search_context("0")); +} + +void +SearchContextTest::single_bool_attribute_search_iterator_handles_true_and_false_queries() +{ + BoolAttributeFixture f(SimpleResult().addHit(3).addHit(5).addHit(7), 9); + + auto true_exp = SimpleResult().addHit(3).addHit(5).addHit(7); + EXPECT_EQUAL(true_exp, f.search_iterator("true", false)); + EXPECT_EQUAL(true_exp, f.search_iterator("1", false)); + EXPECT_EQUAL(true_exp, f.search_iterator("true", true)); + EXPECT_EQUAL(true_exp, f.search_iterator("1", true)); + + auto false_exp = SimpleResult().addHit(1).addHit(2).addHit(4).addHit(6).addHit(8); + EXPECT_EQUAL(false_exp, f.search_iterator("false", false)); + EXPECT_EQUAL(false_exp, f.search_iterator("0", false)); + EXPECT_EQUAL(false_exp, f.search_iterator("false", true)); + EXPECT_EQUAL(false_exp, f.search_iterator("0", true)); +} + void SearchContextTest::initIntegerConfig() { @@ -1955,6 +2044,8 @@ SearchContextTest::Main() TEST_DO(requireThatInvalidSearchTermGivesZeroHits()); TEST_DO(requireThatFlagAttributeHandlesTheByteRange()); TEST_DO(requireThatOutOfBoundsSearchTermGivesZeroHits()); + TEST_DO(single_bool_attribute_search_context_handles_true_and_false_queries()); + TEST_DO(single_bool_attribute_search_iterator_handles_true_and_false_queries()); TEST_DONE(); } diff --git a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp index ed681d9021b..842c42d4189 100644 --- a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp +++ b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp @@ -51,10 +51,10 @@ void BitVectorBenchmark::init(size_t n) BitVector *b(BitVector::create(n).release()); srand(1); for(size_t i(0), j(0); i < n; i += rand()%10, j++) { - a->flip(i); + a->flipBit(i); } for(size_t i(0), j(0); i < n; i += rand()%10, j++) { - b->flip(i); + b->flipBit(i); } a->invalidateCachedCount(); b->invalidateCachedCount(); @@ -73,7 +73,7 @@ void BitVectorBenchmark::testCountSpeed1() { _bv[0]->invalidateCachedCount(); unsigned int cnt = _bv[0]->countTrueBits(); - assert(cnt = _bvc[0]); + assert(cnt == _bvc[0]); (void) cnt; } diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 4cbe96c74b5..409cc9f2725 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -268,21 +268,21 @@ TEST("requireThatSequentialOperationsOnPartialWorks") p1.invalidateCachedCount(); EXPECT_TRUE(p1.hasTrueBits()); EXPECT_EQUAL(1u, p1.countTrueBits()); - p1.slowSetBit(718); - p1.slowSetBit(739); - p1.slowSetBit(871); - p1.slowSetBit(903); + p1.setBitAndMaintainCount(718); + p1.setBitAndMaintainCount(739); + p1.setBitAndMaintainCount(871); + p1.setBitAndMaintainCount(903); EXPECT_EQUAL(5u, p1.countTrueBits()); EXPECT_TRUE(assertBV("[718,719,739,871,903]", p1)); PartialBitVector p2(717,919); EXPECT_FALSE(p1 == p2); - p2.slowSetBit(719); - p2.slowSetBit(718); - p2.slowSetBit(739); - p2.slowSetBit(871); + p2.setBitAndMaintainCount(719); + p2.setBitAndMaintainCount(718); + p2.setBitAndMaintainCount(739); + p2.setBitAndMaintainCount(871); EXPECT_FALSE(p1 == p2); - p2.slowSetBit(903); + p2.setBitAndMaintainCount(903); EXPECT_TRUE(p1 == p2); AllocatedBitVector full(1000); @@ -422,10 +422,10 @@ TEST("requireThatSetWorks") EXPECT_EQUAL(4u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,80,103]", v1)); - v1.slowSetBit(39); + v1.setBitAndMaintainCount(39); EXPECT_EQUAL(4u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,80,103]", v1)); - v1.slowSetBit(57); + v1.setBitAndMaintainCount(57); EXPECT_EQUAL(5u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,57,80,103]", v1)); } @@ -542,15 +542,16 @@ TEST("requireThatGrowWorks") { vespalib::GenerationHolder g; GrowableBitVector v(200, 200, g); + EXPECT_EQUAL(0u, v.countTrueBits()); - v.setBit(7); - v.setBit(39); - v.setBit(71); - v.setBit(103); - - EXPECT_EQUAL(200u, v.size()); + v.setBitAndMaintainCount(7); + v.setBitAndMaintainCount(39); + v.setBitAndMaintainCount(71); + v.setBitAndMaintainCount(103); + EXPECT_EQUAL(4u, v.countTrueBits()); + + EXPECT_EQUAL(200u, v.size()); EXPECT_EQUAL(1023u, v.capacity()); - v.invalidateCachedCount(); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_TRUE(v.reserve(1024)); @@ -584,6 +585,13 @@ TEST("requireThatGrowWorks") EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71]", v)); EXPECT_EQUAL(3u, v.countTrueBits()); + + v.invalidateCachedCount(); + EXPECT_TRUE(v.reserve(3100)); + EXPECT_EQUAL(100u, v.size()); + EXPECT_EQUAL(4095u, v.capacity()); + EXPECT_EQUAL(3u, v.countTrueBits()); + g.transferHoldLists(1); g.trimHoldLists(2); } diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 0a3c3788c98..cc31fcec4d4 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -127,7 +127,7 @@ MyVisitor::visit(uint32_t lid, const std::shared_ptr<Document> &doc) assert(lid < _docIdLimit); Document::UP expDoc(makeDoc(_repo, lid, _before)); EXPECT_TRUE(*expDoc == *doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); } @@ -136,7 +136,7 @@ MyVisitor::visit(uint32_t lid) { ++_visitRmCount; assert(lid < _docIdLimit); - _valid->slowClearBit(lid); + _valid->clearBitAndMaintainCount(lid); } @@ -158,7 +158,7 @@ MyRewriteVisitor::visit(uint32_t lid, const std::shared_ptr<Document> &doc) assert(lid < _docIdLimit); Document::UP expDoc(makeDoc(_repo, lid, _before)); EXPECT_TRUE(*expDoc == *doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); doc->set("extra", "foo"); } @@ -297,7 +297,7 @@ Fixture::put(const Document &doc, uint32_t lid) ++_syncToken; assert(lid < _docIdLimit); _store->write(_syncToken, lid, doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); } @@ -307,7 +307,7 @@ Fixture::remove(uint32_t lid) ++_syncToken; assert(lid < _docIdLimit); _store->remove(_syncToken, lid); - _valid->slowClearBit(lid); + _valid->clearBitAndMaintainCount(lid); } diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 503922bdb13..02d459dd0d8 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -267,7 +267,7 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut { auto node = makeNode(attributeName, false, preserveAccurateTypes); uint32_t docId = 0; - for (const auto &expDocVal : expVals) { + for (const auto expDocVal : expVals) { ++docId; node->setDocId(docId); node->execute(); diff --git a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp index b7fb3d2b6a1..2f710c5d6e1 100644 --- a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp +++ b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp @@ -111,7 +111,7 @@ struct ArrayFixture : FixtureBase { void check_prepare_state_output(const vespalib::tensor::Tensor & tensor, const ExpectedType & expected) { vespalib::nbostream os; vespalib::tensor::TypedBinaryFormat::serialize(os, tensor); - vespalib::string input_vector(os.c_str(), os.size()); + vespalib::string input_vector(os.data(), os.size()); check_prepare_state_output(".tensor", input_vector, expected); } diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp index 896a917d6e3..4d1163de6ee 100644 --- a/searchlib/src/tests/fef/fef_test.cpp +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -94,6 +94,7 @@ TEST("verify size of essential fef classes") { EXPECT_EQUAL(24u,sizeof(TermFieldMatchDataPosition)); EXPECT_EQUAL(24u,sizeof(TermFieldMatchData::Features)); EXPECT_EQUAL(40u,sizeof(TermFieldMatchData)); + EXPECT_EQUAL(48u, sizeof(search::fef::FeatureExecutor)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp index 28ba13bb4fc..12c38d3fe02 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/test/searchiteratorverifier.h> +#include <random> #include <vespa/log/log.h> LOG_SETUP("multibitvectoriterator_test"); @@ -52,6 +53,14 @@ private: BitVector * getBV(size_t index, bool inverted) { return inverted ? _bvs_inverted[index].get() : _bvs[index].get(); } + void fixup_bitvectors() { + // Restore from inverted bitvectors after tampering + for (int i = 0; i < 3; ++i) { + if (_bvs_inverted[i]->testBit(1)) { + _bvs[i]->clearBit(1); + } + } + } std::vector< BitVector::UP > _bvs; std::vector< BitVector::UP > _bvs_inverted; }; @@ -61,12 +70,12 @@ Test::~Test() = default; void Test::setup() { - srand(7); + std::minstd_rand rnd(341); for(size_t i(0); i < 3; i++) { _bvs.push_back(BitVector::create(10000)); BitVector & bv(*_bvs.back()); for (size_t j(0); j < bv.size(); j++) { - int r = rand(); + int r = rnd(); if (r & 0x1) { bv.setBit(j); } @@ -132,7 +141,7 @@ Test::testAndWith(bool invert) s->initFullRange(); H firstHits3 = seekNoReset(*s, 1, 130); H lastHits3 = seekNoReset(*s, 130, _bvs[0]->size()); - //These constants will change if srand(7) is changed. + //These constants will change if rnd(341) is changed. EXPECT_EQUAL(30u, firstHits2.size()); EXPECT_EQUAL(19u, firstHits3.size()); EXPECT_EQUAL(1234u, lastHits2F.size()); @@ -208,6 +217,7 @@ Test::testBug7163266() EXPECT_TRUE(ms->needUnpack(i)); } EXPECT_TRUE(ms->needUnpack(28)); // NB: force unpack all + fixup_bitvectors(); } template<typename T> @@ -240,6 +250,7 @@ Test::testThatOptimizePreservesUnpack() EXPECT_TRUE(ms != nullptr); EXPECT_EQUAL(2u, ms->getChildren().size()); verifySelectiveUnpack(*s, tfmd); + fixup_bitvectors(); } void diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp index 32dc9b1fa7e..5601baa9113 100644 --- a/searchlib/src/tests/queryeval/queryeval.cpp +++ b/searchlib/src/tests/queryeval/queryeval.cpp @@ -346,29 +346,12 @@ public: (void) tfmda; return _sc->createIterator(&_tfmd, strict); } - const search::AttributeVector::SearchContext & getSearchContext() const { return *_sc; } private: search::SingleBoolAttribute _a; search::AttributeVector::SearchContext::UP _sc; mutable TermFieldMatchData _tfmd; }; -TEST("test bool attribute searchcontext") { - SimpleResult a; - a.addHit(5).addHit(17).addHit(30); - auto bp = std::make_unique<DummySingleValueBitNumericAttributeBlueprint>(a); - const search::AttributeVector::SearchContext & sc = bp->getSearchContext(); - EXPECT_FALSE(sc.matches(7)); - EXPECT_TRUE(sc.matches(17)); - int32_t weight(0); - EXPECT_FALSE(sc.matches(7, weight)); - EXPECT_EQUAL(0, weight); - EXPECT_TRUE(sc.matches(17, weight)); - EXPECT_EQUAL(1, weight); - EXPECT_FALSE(sc.matches(27, weight)); - EXPECT_EQUAL(0, weight); -} - TEST("testAndNot") { { diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt new file mode 100644 index 00000000000..04c7312a63f --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_hnsw_index_test_app TEST + SOURCES + hnsw_index_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app) diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp new file mode 100644 index 00000000000..c6246bb8434 --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -0,0 +1,266 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/searchlib/tensor/distance_functions.h> +#include <vespa/searchlib/tensor/doc_vector_access.h> +#include <vespa/searchlib/tensor/hnsw_index.h> +#include <vespa/searchlib/tensor/random_level_generator.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vector> + +#include <vespa/log/log.h> +LOG_SETUP("hnsw_index_test"); + +using namespace search::tensor; + +template <typename FloatType> +class MyDocVectorAccess : public DocVectorAccess { +private: + using Vector = std::vector<FloatType>; + using ArrayRef = vespalib::ConstArrayRef<FloatType>; + std::vector<Vector> _vectors; + +public: + MyDocVectorAccess() : _vectors() {} + MyDocVectorAccess& set(uint32_t docid, const Vector& vec) { + if (docid >= _vectors.size()) { + _vectors.resize(docid + 1); + } + _vectors[docid] = vec; + return *this; + } + vespalib::tensor::TypedCells get_vector(uint32_t docid) const override { + ArrayRef ref(_vectors[docid]); + return vespalib::tensor::TypedCells(ref); + } +}; + +struct LevelGenerator : public RandomLevelGenerator { + uint32_t level; + LevelGenerator() : level(0) {} + uint32_t max_level() override { return level; } +}; + +using FloatVectors = MyDocVectorAccess<float>; +using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>; +using HnswIndexUP = std::unique_ptr<HnswIndex>; + +class HnswIndexTest : public ::testing::Test { +public: + FloatVectors vectors; + FloatSqEuclideanDistance distance_func; + LevelGenerator level_generator; + HnswIndexUP index; + + HnswIndexTest() + : vectors(), + level_generator(), + index() + { + vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) + .set(4, {1, 2}).set(5, {8, 3}).set(6, {7, 2}) + .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5}); + } + void init(bool heuristic_select_neighbors) { + index = std::make_unique<HnswIndex>(vectors, distance_func, level_generator, + HnswIndex::Config(2, 1, 10, heuristic_select_neighbors)); + } + void add_document(uint32_t docid, uint32_t max_level = 0) { + level_generator.level = max_level; + index->add_document(docid); + } + void remove_document(uint32_t docid) { + index->remove_document(docid); + } + void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) { + EXPECT_EQ(exp_docid, index->get_entry_docid()); + EXPECT_EQ(exp_level, index->get_entry_level()); + } + void expect_level_0(uint32_t docid, const HnswNode::LinkArray& exp_links) { + auto node = index->get_node(docid); + ASSERT_EQ(1, node.size()); + EXPECT_EQ(exp_links, node.level(0)); + } + void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) { + auto act_node = index->get_node(docid); + ASSERT_EQ(exp_levels.size(), act_node.size()); + EXPECT_EQ(exp_levels, act_node.levels()); + } + void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) { + uint32_t k = 3; + auto qv = vectors.get_vector(docid); + auto rv = index->top_k_candidates(qv, k).peek(); + std::sort(rv.begin(), rv.end(), LesserDistance()); + size_t idx = 0; + for (const auto & hit : rv) { + if (idx < exp_hits.size()) { + EXPECT_EQ(hit.docid, exp_hits[idx++]); + } + } + if (exp_hits.size() == k) { + std::vector<uint32_t> expected_by_docid = exp_hits; + std::sort(expected_by_docid.begin(), expected_by_docid.end()); + std::vector<uint32_t> got_by_docid = index->find_top_k(k, qv, k); + EXPECT_EQ(expected_by_docid, got_by_docid); + } + } +}; + + +TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_neighbors) +{ + init(false); + + add_document(1); + expect_level_0(1, {}); + + add_document(2); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + + add_document(3); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2}); + + add_document(4); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2, 4}); + expect_level_0(4, {1, 3}); + + add_document(5); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5}); + expect_level_0(3, {1, 2, 4, 5}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3}); + + add_document(6); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_level_0(3, {1, 2, 4, 5}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3, 6}); + expect_level_0(6, {2, 5}); + + add_document(7); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6, 7}); + expect_level_0(3, {1, 2, 4, 5, 7}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3, 6}); + expect_level_0(6, {2, 5}); + expect_level_0(7, {2, 3}); + + expect_top_3(1, {1}); + expect_top_3(2, {2, 1, 3}); + expect_top_3(3, {3}); + expect_top_3(4, {4, 1, 3}); + expect_top_3(5, {5, 6, 2}); + expect_top_3(6, {6, 5, 2}); + expect_top_3(7, {7, 3, 2}); + expect_top_3(8, {4, 3, 1}); + expect_top_3(9, {7, 3, 2}); +} + +TEST_F(HnswIndexTest, 2d_vectors_inserted_and_removed) +{ + init(false); + + add_document(1); + expect_level_0(1, {}); + expect_entry_point(1, 0); + + add_document(2); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + expect_entry_point(1, 0); + + add_document(3); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2}); + expect_entry_point(1, 0); + + remove_document(2); + expect_level_0(1, {3}); + expect_level_0(3, {1}); + expect_entry_point(1, 0); + + remove_document(1); + expect_level_0(3, {}); + expect_entry_point(3, 0); + + remove_document(3); + expect_entry_point(0, -1); +} + +TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_select_neighbors) +{ + init(true); + + add_document(1); + expect_entry_point(1, 0); + expect_level_0(1, {}); + + add_document(2); + expect_entry_point(1, 0); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + + // Doc 3 is also added to level 1 + add_document(3, 1); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_levels(3, {{1, 2}, {}}); + + // Doc 4 is closest to 1 and they are linked. + // Doc 4 is NOT linked to 3 as the distance between 4 and 3 is greater than the distance between 3 and 1. + // Doc 3 is therefore reachable via 1. Same argument for why doc 4 is not linked to 2. + add_document(4); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3}); + expect_levels(3, {{1, 2}, {}}); + expect_level_0(4, {1}); + + // Doc 5 is closest to 2 and they are linked. + // The other docs are reachable via 2, and no other links are created. Same argument as with doc 4 above. + add_document(5); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5}); + expect_levels(3, {{1, 2}, {}}); + expect_level_0(4, {1}); + expect_level_0(5, {2}); + + // Doc 6 is closest to 5 and they are linked. + // Doc 6 is also linked to 2 as the distance between 6 and 2 is less than the distance between 2 and 5. + // Doc 6 is also added to level 1 and 2, and linked to doc 3 in level 1. + add_document(6, 2); + expect_entry_point(6, 2); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_levels(3, {{1, 2}, {6}}); + expect_level_0(4, {1}); + expect_level_0(5, {2, 6}); + expect_levels(6, {{2, 5}, {3}, {}}); + + // Doc 7 is closest to 3 and they are linked. + // Doc 7 is also linked to 6 as the distance between 7 and 6 is less than the distance between 6 and 3. + // Docs 1, 2, 4 are reachable via 3. + add_document(7); + expect_entry_point(6, 2); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_levels(3, {{1, 2, 7}, {6}}); + expect_level_0(4, {1}); + expect_level_0(5, {2, 6}); + expect_levels(6, {{2, 5, 7}, {3}, {}}); + expect_level_0(7, {3, 6}); +} + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index c4751af5adb..0dced597917 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -63,27 +63,27 @@ class CallBackTest : public TransLogClient::Visitor::Callback private: virtual RPC::Result receive(const Packet & packet) override; virtual void eof() override { _eof = true; } - typedef std::map<SerialNum, ByteBuffer> PacketMap; + typedef std::map<SerialNum, std::unique_ptr<ByteBuffer>> PacketMap; PacketMap _packetMap; public: CallBackTest() : _eof(false) { } size_t size() const { return _packetMap.size(); } bool hasSerial(SerialNum n) const { return (_packetMap.find(n) != _packetMap.end()); } void clear() { _eof = false; _packetMap.clear(); } - const ByteBuffer & packet(SerialNum n) { return (_packetMap.find(n)->second); } + const ByteBuffer & packet(SerialNum n) { return *(_packetMap.find(n)->second); } bool _eof; }; RPC::Result CallBackTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(h.peek(), h.size()).c_str()); while(h.size() > 0) { Packet::Entry e; e.deserialize(h); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(e.data().c_str(), e.data().size()).c_str()); - _packetMap[e.serial()] = ByteBuffer(e.data().c_str(), e.data().size()); + _packetMap[e.serial()] = std::make_unique<ByteBuffer>(e.data().c_str(), e.data().size()); } return RPC::OK; } @@ -103,7 +103,7 @@ public: RPC::Result CallBackManyTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; _count++, _value++) { Packet::Entry e; e.deserialize(h); @@ -135,7 +135,7 @@ public: RPC::Result CallBackUpdate::receive(const Packet & packet) { - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); while (h.size() > 0) { Packet::Entry e; e.deserialize(h); @@ -187,7 +187,7 @@ public: RPC::Result CallBackStatsTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; ++_count) { Packet::Entry e; e.deserialize(h); @@ -236,13 +236,13 @@ bool Test::partialUpdateTest() nbostream os; os << du; - vespalib::ConstBufferRef bb(os.c_str(), os.size()); + vespalib::ConstBufferRef bb(os.data(), os.size()); LOG(info, "DU : %s", myhex(bb.c_str(), bb.size()).c_str()); Packet::Entry e(7, du.getClass().id(), bb); Packet pa; pa.add(e); pa.close(); - ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().c_str(), pa.getHandle().size()))); + ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().data(), pa.getHandle().size()))); CallBackUpdate ca; TransLogClient::Visitor::UP visitor = tls.createVisitor("test1", ca); @@ -320,10 +320,10 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & ASSERT_TRUE (!b.add(e1)); a.close(); b.close(); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size()))); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().c_str(), b.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().data(), b.getHandle().size()))); try { - s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size())); + s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size())); ASSERT_TRUE(false); } catch (const std::exception & e) { EXPECT_EQUAL(vespalib::string("commit failed with code -2. server says: Exception during commit on " + name + " : Incomming serial number(1) must be bigger than the last one (3)."), e.what()); @@ -340,7 +340,7 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & EXPECT_EQUAL(a.range().to(), 3u); Packet::Entry e; - vespalib::nbostream h(a.getHandle().c_str(), a.getHandle().size()); + vespalib::nbostream h(a.getHandle().data(), a.getHandle().size()); e.deserialize(h); e.deserialize(h); e.deserialize(h); @@ -358,13 +358,13 @@ void Test::fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_ Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } @@ -382,13 +382,13 @@ Test::fillDomainTest(TransLogClient::Session * s1, Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&entryBuffer[0], entryBuffer.size())); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index a0e4b4884a9..74e48081e17 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -17,7 +17,7 @@ LOG_SETUP("translogstress"); -using document::ByteBuffer; +using vespalib::nbostream; using search::Runnable; using vespalib::Monitor; using vespalib::MonitorGuard; @@ -47,10 +47,10 @@ public: BufferGenerator(uint32_t minStrLen, uint32_t maxStrLen) : _rnd(), _minStrLen(minStrLen), _maxStrLen(maxStrLen) {} void setSeed(long seed) { _rnd.srand48(seed); } - ByteBuffer getRandomBuffer(); + nbostream getRandomBuffer(); }; -ByteBuffer +nbostream BufferGenerator::getRandomBuffer() { size_t len = _minStrLen + _rnd.lrand48() % (_maxStrLen - _minStrLen); @@ -59,9 +59,8 @@ BufferGenerator::getRandomBuffer() char c = 'a' + _rnd.lrand48() % ('z' - 'a' + 1); str.push_back(c); } - ByteBuffer buf(str.size() + 1); - buf.putBytes(str.c_str(), str.size() + 1); - buf.flip(); + nbostream buf(str.size() + 1); + buf.write(str.c_str(), str.size() + 1); return buf; } @@ -75,13 +74,13 @@ private: Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; - const std::vector<document::ByteBuffer> * _buffers; - ByteBuffer _lastGeneratedBuffer; + const std::vector<nbostream> * _buffers; + nbostream _lastGeneratedBuffer; public: EntryGenerator(long baseSeed, const BufferGenerator & bufferGenerator) : - _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(NULL), - _lastGeneratedBuffer() {} + _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(nullptr), + _lastGeneratedBuffer(0) {} EntryGenerator(const EntryGenerator & rhs) : _rnd(), _baseSeed(rhs._baseSeed), _bufferGenerator(rhs._bufferGenerator), _buffers(rhs._buffers), _lastGeneratedBuffer(rhs._lastGeneratedBuffer) {} @@ -95,7 +94,7 @@ public: SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); Rand48 & getRnd() { return _rnd; } - void setBuffers(const std::vector<ByteBuffer> & buffers) { + void setBuffers(const std::vector<nbostream> & buffers) { _buffers = &buffers; } }; @@ -118,12 +117,12 @@ EntryGenerator::getRandomEntry(SerialNum num) _rnd.srand48(_baseSeed + num); if (_buffers != NULL) { size_t i = _rnd.lrand48() % _buffers->size(); - const ByteBuffer& buffer = (*_buffers)[i]; - return Packet::Entry(num, 1024, ConstBufferRef(buffer.getBuffer(), buffer.getLength())); + const nbostream& buffer = (*_buffers)[i]; + return Packet::Entry(num, 1024, ConstBufferRef(buffer.data(), buffer.size())); } else { _bufferGenerator.setSeed(_baseSeed + num); _lastGeneratedBuffer = _bufferGenerator.getRandomBuffer(); - return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.getBuffer(), _lastGeneratedBuffer.getLength())); + return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.data(), _lastGeneratedBuffer.size())); } } @@ -226,7 +225,7 @@ FeederThread::commitPacket() { _packet.close(); const vespalib::nbostream& stream = _packet.getHandle(); - if (!_session->commit(ConstBufferRef(stream.c_str(), stream.size()))) { + if (!_session->commit(ConstBufferRef(stream.data(), stream.size()))) { throw std::runtime_error(vespalib::make_string ("FeederThread: Failed commiting %s", PacketPrinter::toStr(_packet).c_str())); } else { @@ -708,7 +707,7 @@ TransLogStress::Main() BufferGenerator bufferGenerator(_cfg.minStrLen, _cfg.maxStrLen); bufferGenerator.setSeed(_cfg.baseSeed); - std::vector<ByteBuffer> buffers; + std::vector<nbostream> buffers; for (uint32_t i = 0; i < _cfg.numPreGeneratedBuffers; ++i) { buffers.push_back(bufferGenerator.getRandomBuffer()); } diff --git a/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp b/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp index 0efe2adfaf9..2cc82d59fff 100644 --- a/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp @@ -3,6 +3,7 @@ #include "hitlist.h" #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> +#include <stdexcept> namespace search::aggregation { diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index da280aa7bda..5fa32e8ad75 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -131,7 +131,7 @@ bool AttributeManager::hasReaders() const const AttributeManager::VectorHolder * AttributeManager::findAndLoadAttribute(const string & name) const { - const VectorHolder * loadedVector(NULL); + const VectorHolder * loadedVector(nullptr); AttributeMap::const_iterator found = _attributes.find(name); if (found != _attributes.end()) { AttributeVector & vec = *found->second; @@ -263,5 +263,14 @@ AttributeManager::asyncForAttribute(const vespalib::string &, std::unique_ptr<at throw std::runtime_error("search::AttributeManager::asyncForAttribute should never be called."); } +std::shared_ptr<attribute::ReadableAttributeVector> +AttributeManager::readable_attribute_vector(const string& name) const +{ + const auto* attr = findAndLoadAttribute(name); + if (attr) { + return *attr; + } + return {}; +} } diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h index 1c60ab00585..9f30ff87a70 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h @@ -49,6 +49,8 @@ public: void getAttributeList(AttributeList & list) const override; attribute::IAttributeContext::UP createContext() const override; + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; + const Snapshot & getSnapshot() const { return _snapShot; } const string & getBaseDir() const { return _baseDir; } void setBaseDir(const string & base); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index a38cbe60e56..1e4bba95b4b 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -24,8 +24,7 @@ class SaveBits FA &_fa; public: - SaveBits(vespalib::ConstArrayRef<T> map, - FA &fa) + SaveBits(vespalib::ConstArrayRef<T> map, FA &fa) : _map(map), _fa(fa) { @@ -55,11 +54,9 @@ FlagAttributeT<B>::FlagAttributeT(const vespalib::string & baseFileName, const A template <typename B> AttributeVector::SearchContext::UP -FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, - const attribute::SearchContextParams & params) const +FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams &) const { - (void) params; - return AttributeVector::SearchContext::UP (new SearchContext(std::move(qTerm), *this)); + return std::make_unique<SearchContext>(std::move(qTerm), *this); } template <typename B> @@ -69,7 +66,7 @@ void FlagAttributeT<B>::clearOldValues(DocId doc) for (uint32_t i(0), m(this->get(doc, values)); i < m; i++) { BitVector * bv = _bitVectors[getOffset(values[i].value())]; if (bv != nullptr) { - bv->clearBit(doc); + bv->clearBitAndMaintainCount(doc); } } } @@ -130,10 +127,9 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT _bitVectorStore[offset] = BitVector::create(_bitVectorSize); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; - bv->invalidateCachedCount(); ensureGuardBit(*bv); } - bv->setBit(doc); + bv->setBitAndMaintainCount(doc); } } @@ -145,13 +141,12 @@ FlagAttributeT<B>::setNewBVValue(DocId doc, typename B::WType::ValueType value) BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = BitVector::create(_bitVectorSize); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; - bv->invalidateCachedCount(); ensureGuardBit(*bv); } - bv->setBit(doc); + bv->setBitAndMaintainCount(doc); } @@ -193,8 +188,7 @@ template <typename B> void FlagAttributeT<B>::ensureGuardBit() { - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { ensureGuardBit(*bv); } @@ -205,8 +199,7 @@ template <typename B> void FlagAttributeT<B>::clearGuardBit(DocId doc) { - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { bv->clearBit(doc); // clear guard bit and start using this doc id } @@ -219,8 +212,7 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { vespalib::GenerationHeldBase::UP hold(bv->grow(newSize)); ensureGuardBit(*bv); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index 742d0cd8592..d3ce5f9af46 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -52,10 +52,10 @@ private: return _bitVectors[value + 128]; } - vespalib::GenerationHolder _bitVectorHolder; + vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<BitVector> > _bitVectorStore; std::vector<BitVector *> _bitVectors; - uint32_t _bitVectorSize; + uint32_t _bitVectorSize; template <class SC> friend class FlagAttributeIteratorT; template <class SC> friend class FlagAttributeIteratorStrict; }; diff --git a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h index ef9403e1c4b..51e432f2035 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h @@ -8,7 +8,10 @@ namespace search { -namespace attribute { class AttributeReadGuard; } +namespace attribute { +class AttributeReadGuard; +class ReadableAttributeVector; +} /** * This is an interface used to access all registered attribute vectors. @@ -17,12 +20,17 @@ class IAttributeManager : public attribute::IAttributeExecutor { public: IAttributeManager(const IAttributeManager &) = delete; IAttributeManager & operator = (const IAttributeManager &) = delete; - typedef std::shared_ptr<IAttributeManager> SP; - typedef vespalib::string string; + using SP = std::shared_ptr<IAttributeManager>; + using string = vespalib::string; /** * Returns a view of the attribute vector with the given name. * + * NOTE: this method is deprecated! Prefer using readable_attribute_vector(name) instead, + * as that enforces appropriate guards to be taken before accessing the underlying vector. + * + * TODO remove this when all usages are gone. + * * @param name name of the attribute vector. * @return view of the attribute vector or empty view if the attribute vector does not exists. **/ @@ -52,9 +60,15 @@ public: virtual attribute::IAttributeContext::UP createContext() const = 0; /** - * Virtual destructor to allow safe subclassing. - **/ - virtual ~IAttributeManager() {} + * Looks up and returns a readable attribute vector shared_ptr with the provided name. + * This transparently supports imported attribute vectors. + * + * @param name name of the attribute vector. + * @return The attribute vector, or an empty shared_ptr if no vector was found with the given name. + */ + virtual std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const = 0; + + ~IAttributeManager() override = default; protected: IAttributeManager() = default; }; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index b097b27ea53..f2949119ae2 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -152,6 +152,10 @@ bool ImportedAttributeVectorReadGuard::isImported() const return true; } +bool ImportedAttributeVectorReadGuard::isUndefined(DocId doc) const { + return _target_attribute.isUndefined(getTargetLid(doc)); +} + long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc, void *serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index f130a095006..b31b11b8c74 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -81,6 +81,7 @@ public: bool getIsFastSearch() const override; uint32_t getCommittedDocIdLimit() const override; bool isImported() const override; + bool isUndefined(DocId doc) const override; protected: long onSerializeForAscendingSort(DocId doc, void * serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/iterator_pack.h b/searchlib/src/vespa/searchlib/attribute/iterator_pack.h index e346ae66eb7..579f8428088 100644 --- a/searchlib/src/vespa/searchlib/attribute/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/attribute/iterator_pack.h @@ -16,13 +16,8 @@ private: public: AttributeIteratorPack() : _children() {} - AttributeIteratorPack(AttributeIteratorPack &&rhs) - : _children(std::move(rhs._children)) {} - - AttributeIteratorPack &operator=(AttributeIteratorPack &&rhs) { - _children = std::move(rhs._children); - return *this; - } + AttributeIteratorPack(AttributeIteratorPack &&rhs) noexcept = default; + AttributeIteratorPack &operator=(AttributeIteratorPack &&rhs) noexcept = default; explicit AttributeIteratorPack(std::vector<DocumentWeightIterator> &&children) : _children(std::move(children)) {} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 3dcc0df477d..07f8077ae97 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -42,9 +42,9 @@ template <typename EntryT, typename RefT> void MultiValueMapping<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values) { - ConstArrayRef oldValues = _store.get(_indices[docId]); + auto oldValues = _store.get_writable(_indices[docId]); assert(oldValues.size() == values.size()); - EntryT *dst = const_cast<EntryT *>(&oldValues[0]); + EntryT *dst = &oldValues[0]; for (auto &src : values) { *dst = src; ++dst; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 9c736a16069..2badb199934 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -246,9 +246,8 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) uint32_t typeId = getTypeId(iRef); assert(isBTree(typeId)); (void) typeId; - std::shared_ptr<GrowableBitVector> bvsp; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); - bvsp.reset(new GrowableBitVector(_bvSize, _bvCapacity, genHolder)); + auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); AllocatedBitVector &bv = *bvsp.get(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; @@ -288,9 +287,8 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, { assert(!ref.valid()); RefType iRef(ref); - std::shared_ptr<GrowableBitVector> bvsp; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); - bvsp.reset(new GrowableBitVector(_bvSize, _bvCapacity, genHolder)); + auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); AllocatedBitVector &bv = *bvsp.get(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; @@ -329,17 +327,17 @@ PostingStore<DataT>::apply(BitVector &bv, if (r != re && (a == ae || *r < a->_key)) { // remove assert(*r < bv.size()); - bv.slowClearBit(*r); + bv.clearBitAndMaintainCount(*r); ++r; } else { if (r != re && !(a->_key < *r)) { // update or add assert(a->_key < bv.size()); - bv.slowSetBit(a->_key); + bv.setBitAndMaintainCount(a->_key); ++r; } else { assert(a->_key < bv.size()); - bv.slowSetBit(a->_key); + bv.setBitAndMaintainCount(a->_key); } ++a; } diff --git a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h index 03058362cb1..0e3954bdcb5 100644 --- a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h +++ b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h @@ -13,7 +13,7 @@ class AttributeReadGuard; */ class ReadableAttributeVector { public: - virtual ~ReadableAttributeVector() {} + virtual ~ReadableAttributeVector() = default; virtual std::unique_ptr<AttributeReadGuard> makeReadGuard(bool stableEnumGuard) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 250182f924b..e1ab47ed434 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -60,25 +60,16 @@ SingleBoolAttribute::onCommit() { for (const auto & change : _changes) { if (change._type == ChangeBase::UPDATE) { std::atomic_thread_fence(std::memory_order_release); - if (change._data == 0) { - _bv.clearBit(change._doc); - } else { - _bv.setBit(change._doc); - } + setBit(change._doc, change._data != 0); } else if ((change._type >= ChangeBase::ADD) && (change._type <= ChangeBase::DIV)) { std::atomic_thread_fence(std::memory_order_release); int8_t val = applyArithmetic(getFast(change._doc), change); - if (val == 0) { - _bv.clearBit(change._doc); - } else { - _bv.setBit(change._doc); - } + setBit(change._doc, val != 0); } else if (change._type == ChangeBase::CLEARDOC) { std::atomic_thread_fence(std::memory_order_release); - _bv.clearBit(change._doc); + _bv.clearBitAndMaintainCount(change._doc); } } - _bv.invalidateCachedCount(); } std::atomic_thread_fence(std::memory_order_release); @@ -113,7 +104,7 @@ private: bool _valid; bool valid() const override { return _valid; } int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override final { - if ((elemId == 0) && _bv.testBit(docId)) { + if ((elemId == 0) && (_invert != _bv.testBit(docId))) { weight = 1; return 0; } @@ -122,7 +113,7 @@ private: } int32_t onFind(DocId docId, int32_t elemId) const override final { - return ((elemId == 0) && _bv.testBit(docId)) ? 0 : -1; + return ((elemId == 0) && (_invert != _bv.testBit(docId))) ? 0 : -1; } public: @@ -160,8 +151,7 @@ BitVectorSearchContext::createFilterIterator(fef::TermFieldMatchData * matchData } void -BitVectorSearchContext::fetchPostings(const queryeval::ExecuteInfo &execInfo) { - (void) execInfo; +BitVectorSearchContext::fetchPostings(const queryeval::ExecuteInfo &) { } std::unique_ptr<queryeval::SearchIterator> @@ -172,7 +162,9 @@ BitVectorSearchContext::createPostingIterator(fef::TermFieldMatchData *matchData unsigned int BitVectorSearchContext::approximateHits() const { return valid() - ? (_invert) ? (_bv.size() - _bv.countTrueBits()) : _bv.countTrueBits() + ? (_invert) + ? (_bv.size() - _bv.countTrueBits()) + : _bv.countTrueBits() : 0; } @@ -195,6 +187,8 @@ SingleBoolAttribute::onLoad() uint32_t numDocs = attrReader.getNextData(); _bv.extend(numDocs); ssize_t bytesRead = attrReader.getReader().read(_bv.getStart(), _bv.sizeBytes()); + _bv.invalidateCachedCount(); + _bv.countTrueBits(); assert(bytesRead == _bv.sizeBytes()); setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 20ec0b6d077..78c297d55c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -91,9 +91,9 @@ public: const BitVector & getBitVector() const { return _bv; } void setBit(DocId doc, bool value) { if (value) { - _bv.setBit(doc); + _bv.setBitAndMaintainCount(doc); } else { - _bv.clearBit(doc); + _bv.clearBitAndMaintainCount(doc); } } protected: diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index 3de0c1f1320..08a3847f00d 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -52,6 +52,7 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBit } setBit(size()); // Guard bit } + updateCount(); } AllocatedBitVector::AllocatedBitVector(const AllocatedBitVector & rhs) : @@ -70,6 +71,8 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : _capacityBits = computeCapacity(_capacityBits, _alloc.size()); memcpy(_alloc.get(), rhs.getStart(), rhs.sizeBytes()); init(_alloc.get(), 0, rhs.size()); + setBit(size()); + updateCount(); } ////////////////////////////////////////////////////////////////////// @@ -121,12 +124,9 @@ AllocatedBitVector::grow(Index newSize, Index newCapacity) if (newCapacity != capacity()) { AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size()); if (newSize > size()) { - tbv.clearBit(size()); // Clear old guard bit. - } - ret.reset(new GenerationHeldAlloc<Alloc>(_alloc)); - if (( newSize >= size()) && isValidCount()) { - tbv.setTrueBits(countTrueBits()); + tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. } + ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); swap(tbv); } else { if (newSize > size()) { diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 7296842f2c1..e28ebe6682f 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -136,16 +136,6 @@ BitVector::count() const } BitVector::Index -BitVector::internalCount(const Word *tarr, size_t sz) -{ - Index count(0); - for (size_t i(0); i < sz; i++) { - count += Optimized::popCount(tarr[i]); - } - return count; -} - -BitVector::Index BitVector::countInterval(Index start, Index end) const { if (start >= end) return 0; @@ -175,7 +165,7 @@ BitVector::countInterval(Index start, Index end) const ++endw; } if (startw < endw) { - res += internalCount(bitValues + startw, endw - startw); + res += IAccelrated::getAccelrator()->populationCount(bitValues + startw, endw - startw); } if (partialEnd) { res += Optimized::popCount(bitValues[endw] & ~endBits(last)); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 7405688f4f7..9671e41df24 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -29,7 +29,7 @@ public: typedef std::unique_ptr<BitVector> UP; BitVector(const BitVector &) = delete; BitVector& operator = (const BitVector &) = delete; - virtual ~BitVector() { } + virtual ~BitVector() = default; bool operator == (const BitVector &right) const; const void * getStart() const { return _words; } void * getStart() { return _words; } @@ -45,9 +45,9 @@ public: } Index countTrueBits() const { if ( ! isValidCount()) { - _numTrueBits = count(); + updateCount(); } - return _numTrueBits; + return _numTrueBits.load(std::memory_order_relaxed); } /** @@ -134,17 +134,9 @@ public: void clearBit(Index idx) { _words[wordNum(idx)] &= ~ mask(idx); } - void flip(Index idx) { + void flipBit(Index idx) { _words[wordNum(idx)] ^= mask(idx); } - void slowSetBit(Index idx) { - if ( ! testBit(idx) ) { - setBit(idx); - if ( isValidCount() ) { - _numTrueBits++; - } - } - } void andWith(const BitVector &right); void orWith(const BitVector &right); @@ -171,12 +163,24 @@ public: */ void setInterval(Index start, Index end); - void slowClearBit(Index idx) { + /** + * Sets a bit and maintains count of number of bits set. + * @param idx + */ + void setBitAndMaintainCount(Index idx) { + if ( ! testBit(idx) ) { + setBit(idx); + incNumBits(); + } + } + /** + * Clears a bit and maintains count of number of bits set. + * @param idx + */ + void clearBitAndMaintainCount(Index idx) { if (testBit(idx)) { clearBit(idx); - if ( isValidCount() ) { - _numTrueBits--; - } + decNumBits(); } } @@ -185,14 +189,16 @@ public: * should be called before calling Test/Clear/Flip methods. */ void invalidateCachedCount() const { - _numTrueBits = invalidCount(); + _numTrueBits.store(invalidCount(), std::memory_order_relaxed); } void swap(BitVector & rhs) { std::swap(_words, rhs._words); std::swap(_startOffset, rhs._startOffset); std::swap(_sz, rhs._sz); - std::swap(_numTrueBits, rhs._numTrueBits); + Index tmp = rhs._numTrueBits; + rhs._numTrueBits = _numTrueBits.load(std::memory_order_relaxed); + _numTrueBits.store(tmp, std::memory_order_relaxed); } /** @@ -249,9 +255,10 @@ protected: BitVector(void * buf, Index sz) : BitVector(buf, 0, sz) { } BitVector() : BitVector(nullptr, 0) { } void init(void * buf, Index start, Index end); - void setTrueBits(Index numTrueBits) { _numTrueBits = numTrueBits; } + void updateCount() const { _numTrueBits.store(count(), std::memory_order_relaxed); } + void setTrueBits(Index numTrueBits) { _numTrueBits.store(numTrueBits, std::memory_order_relaxed); } VESPA_DLL_LOCAL void clearIntervalNoInvalidation(Index start, Index end); - bool isValidCount() const { return isValidCount(_numTrueBits); } + bool isValidCount() const { return isValidCount(_numTrueBits.load(std::memory_order_relaxed)); } static bool isValidCount(Index v) { return v != invalidCount(); } static Index numWords(Index bits) { return wordNum(bits + 1 + (WordLen - 1)); } static Index numBytes(Index bits) { return numWords(bits) * sizeof(Word); } @@ -279,8 +286,18 @@ private: static size_t numActiveWords(Index start, Index end) { return (numWords(end) - wordNum(start)); } static Index invalidCount() { return std::numeric_limits<Index>::max(); } void setGuardBit() { setBit(size()); } + void incNumBits() { + if ( isValidCount() ) { + _numTrueBits.store(_numTrueBits.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + } + } + void decNumBits() { + if ( isValidCount() ) { + _numTrueBits.store(_numTrueBits.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + + } + } VESPA_DLL_LOCAL void repairEnds(); - VESPA_DLL_LOCAL static Index internalCount(const Word *tarr, size_t sz); Index count() const; bool hasTrueBitsInternal() const; template <typename FunctionType, typename WordConverter> @@ -324,7 +341,7 @@ private: func(start+pos); start += pos + 1; word >>= pos; - word >>= 1; + word >>= 1u; } } @@ -332,7 +349,7 @@ private: Word *_words; // This is the buffer staring at Index 0 Index _startOffset; // This is the official start Index _sz; // This is the official end. - mutable Index _numTrueBits; + mutable std::atomic<Index> _numTrueBits; protected: friend vespalib::nbostream & @@ -341,8 +358,6 @@ protected: operator>>(vespalib::nbostream &in, BitVector &bv); }; -typedef BitVector ConstBitVectorReference; - vespalib::nbostream & operator<<(vespalib::nbostream &out, const BitVector &bv); diff --git a/searchlib/src/vespa/searchlib/common/location.h b/searchlib/src/vespa/searchlib/common/location.h index 96821e204e2..616bf577f0a 100644 --- a/searchlib/src/vespa/searchlib/common/location.h +++ b/searchlib/src/vespa/searchlib/common/location.h @@ -2,12 +2,12 @@ #pragma once -#include <vespa/vespalib/geo/zcurve.h> #include "documentlocations.h" +#include <vespa/vespalib/geo/zcurve.h> + #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace common { +namespace search::common { class Location : public DocumentLocations { @@ -51,5 +51,3 @@ private: }; } -} - diff --git a/searchlib/src/vespa/searchlib/common/sortspec.cpp b/searchlib/src/vespa/searchlib/common/sortspec.cpp index 694443b00ba..f6ac468b485 100644 --- a/searchlib/src/vespa/searchlib/common/sortspec.cpp +++ b/searchlib/src/vespa/searchlib/common/sortspec.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fastlib/text/normwordfolder.h> #include <vespa/vespalib/text/utf8.h> +#include <stdexcept> namespace search { namespace common { diff --git a/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h b/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h index e282a85b64f..3ef081a2820 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h +++ b/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h @@ -5,6 +5,7 @@ #include <vector> #include <unordered_map> #include <limits> +#include <cstdint> namespace search::index { class DocIdAndFeatures; } diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 4707e8001a8..847cd3623c3 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -79,7 +79,7 @@ Chunk::Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc) : os >> _lastSerial; } -Chunk::~Chunk() { } +Chunk::~Chunk() = default; vespalib::ConstBufferRef Chunk::getLid(uint32_t lid) const @@ -89,13 +89,13 @@ Chunk::getLid(uint32_t lid) const if (it->getLid() == lid) { #if 1 uint32_t bLid(0), bLen(0); - vespalib::nbostream is(getData().c_str()+it->getOffset(), it->size()); + vespalib::nbostream is(getData().data() + it->getOffset(), it->size()); is >> bLid >> bLen; assert(bLid == lid); assert(bLen == it->netSize()); assert((bLen + 2*sizeof(uint32_t)) == it->size()); #endif - buf = vespalib::ConstBufferRef(getData().c_str() + it->getNetOffset(), it->netSize()); + buf = vespalib::ConstBufferRef(getData().data() + it->getNetOffset(), it->netSize()); } } return buf; diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 4d76b3fea25..a5bed4c33ce 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -33,7 +33,7 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const const size_t oldPos(compressed.getDataLen()); compressed.writeInt8(compression.type); compressed.writeInt32(os.size()); - CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.c_str(), os.size()), compressed, false)); + CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.data(), os.size()), compressed, false)); if (compression.type != type) { compressed.getData()[oldPos] = type; } diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 4c4405fabb0..6644533550f 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -42,12 +42,6 @@ DocumentVisitorAdapter::visit(uint32_t lid, vespalib::ConstBufferRef buf) { } } -document::Document::UP -deserializeDocument(const vespalib::DataBuffer & uncompressed, const DocumentTypeRepo &repo) { - vespalib::nbostream is(uncompressed.getData(), uncompressed.getDataLen()); - return std::make_unique<document::Document>(repo, is); -} - } using vespalib::nbostream; @@ -183,7 +177,7 @@ DocumentStore::read(DocumentIdT lid, const DocumentTypeRepo &repo) const } Value::Result result = value.decompressed(); if ( result.second ) { - return deserializeDocument(result.first, repo); + return std::make_unique<document::Document>(repo, std::move(result.first)); } else { LOG(warning, "Summary cache for lid %u is corrupt. Invalidating and reading directly from backing store", lid); _cache->invalidate(lid); @@ -195,7 +189,7 @@ DocumentStore::read(DocumentIdT lid, const DocumentTypeRepo &repo) const if ( ! value.empty() ) { Value::Result result = value.decompressed(); assert(result.second); - return deserializeDocument(result.first, repo); + return std::make_unique<document::Document>(repo, std::move(result.first)); } return std::unique_ptr<document::Document>(); } @@ -309,7 +303,7 @@ public: _visitorProgress.updateProgress(progress); } - WrapVisitorProgress(IDocumentStoreVisitorProgress &visitProgress) + explicit WrapVisitorProgress(IDocumentStoreVisitorProgress &visitProgress) : _visitorProgress(visitProgress) { } @@ -369,7 +363,7 @@ DocumentStore::WrapVisitor<Visitor>::visit(uint32_t lid, const void *buffer, siz value.set(std::move(buf), len); } if (! value.empty()) { - std::shared_ptr<document::Document> doc(deserializeDocument(value.decompressed().first, _repo)); + auto doc = std::make_shared<document::Document>(_repo, value.decompressed().first); _visitor.visit(lid, doc); rewrite(lid, *doc); } else { diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 994df3237f2..e8504480b7d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -66,7 +66,7 @@ BlobSet::get(uint32_t lid) const ConstBufferRef buf; for (LidPosition pos : _positions) { if (pos.lid() == lid) { - buf = ConstBufferRef(_buffer.c_str() + pos.offset(), pos.size()); + buf = ConstBufferRef(_buffer.data() + pos.offset(), pos.size()); break; } } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index eb035ac2a2c..8a06794ee35 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -60,7 +60,7 @@ public: void remove(uint32_t lid); const Positions & getPositions() const { return _positions; } vespalib::ConstBufferRef get(uint32_t lid) const; - vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.c_str(), _buffer.size()); } + vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.data(), _buffer.size()); } private: Positions _positions; vespalib::nbostream _buffer; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index e802ec0a326..d12a3a54a93 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -203,6 +203,13 @@ seek_past(LidInfoWithLidV::const_iterator begin, LidInfoWithLidV::const_iterator return begin; } +struct LidAndBuffer { + LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) : _lid(lid), _size(sz), _buf(std::move(buf)) {} + uint32_t _lid; + uint32_t _size; + vespalib::alloc::Alloc _buf; +}; + } void @@ -211,6 +218,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB if (count == 0) { return; } if (!frozen()) { vespalib::hash_map<uint32_t, ChunkInfo> chunksOnFile; + std::vector<LidAndBuffer> buffers; { LockGuard guard(_lock); for (size_t i(0); i < count; i++) { @@ -225,12 +233,18 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB assert(chunk == _active->getId()); buffer = _active->getLid(li.getLid()); } - visitor.visit(li.getLid(), buffer); + auto copy = vespalib::alloc::Alloc::alloc(buffer.size()); + memcpy(copy.get(), buffer.data(), buffer.size()); + buffers.emplace_back(li.getLid(), buffer.size(), std::move(copy)); } else { chunksOnFile[chunk] = _chunkInfo[chunk]; } } } + for (auto & entry : buffers) { + visitor.visit(entry._lid, vespalib::ConstBufferRef(entry._buf.get(), entry._size)); + entry._buf = vespalib::alloc::Alloc(); + } for (auto & it : chunksOnFile) { auto first = find_first(begin, it.first); auto last = seek_past(first, begin + count, it.first); @@ -867,13 +881,13 @@ WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard _pendingDat -= pc.getDataLen(); lastSerial = pc.getLastSerial(); const nbostream &os2(pc.getSerializedIdx()); - os.write(os2.c_str(), os2.size()); + os.write(os2.data(), os2.size()); } } vespalib::system_time timeStamp(vespalib::system_clock::now()); auto idxFile = openIdx(); idxFile->SetPosition(idxFile->GetSize()); - ssize_t wlen = idxFile->Write2(os.c_str(), os.size()); + ssize_t wlen = idxFile->Write2(os.data(), os.size()); updateCurrentDiskFootprint(); if (wlen != static_cast<ssize_t>(os.size())) { diff --git a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp index 4c36960c754..1962cad870a 100644 --- a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "aggregationrefnode.h" #include <vespa/vespalib/util/stringfmt.h> +#include <stdexcept> namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp index 0eb85cba4ba..b770b9ffc9d 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp @@ -489,7 +489,7 @@ bool CatFunctionNode::onExecute() const getArg(i).execute(); getArg(i).getResult().serialize(nos); } - static_cast<RawResultNode &>(updateResult()).setBuffer(os.c_str(), os.size()); + static_cast<RawResultNode &>(updateResult()).setBuffer(os.data(), os.size()); return true; } @@ -520,14 +520,14 @@ bool XorBitFunctionNode::internalExecute(const nbostream & os) const { const size_t numBytes(_tmpXor.size()); memset(&_tmpXor[0], 0, numBytes); - const char * s(os.c_str()); + const char * s(os.data()); for (size_t i(0), m(os.size()/numBytes); i < m; i++) { for (size_t j(0), k(numBytes); j < k; j++) { _tmpXor[j] ^= s[j + k*i]; } } for (size_t i((os.size()/numBytes)*numBytes); i < os.size(); i++) { - _tmpXor[i%numBytes] = os.c_str()[i]; + _tmpXor[i%numBytes] = os.data()[i]; } static_cast<RawResultNode &>(updateResult()).setBuffer(&_tmpXor[0], numBytes); return true; @@ -537,7 +537,7 @@ bool MD5BitFunctionNode::internalExecute(const nbostream & os) const { const unsigned int MD5_DIGEST_LENGTH = 16; unsigned char md5ScratchPad[MD5_DIGEST_LENGTH]; - fastc_md5sum(os.c_str(), os.size(), md5ScratchPad); + fastc_md5sum(os.data(), os.size(), md5ScratchPad); static_cast<RawResultNode &>(updateResult()).setBuffer(md5ScratchPad, std::min(sizeof(md5ScratchPad), getNumBytes())); return true; } diff --git a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp index eea179f67ff..5b6989b5dc9 100644 --- a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "numericfunctionnode.h" +#include <stdexcept> namespace search::expression { diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.cpp b/searchlib/src/vespa/searchlib/expression/resultnode.cpp index 66ac81902d8..a99ac01ce2a 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnode.cpp @@ -3,6 +3,7 @@ #include "resultnode.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exception.h> +#include <stdexcept> namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp index fb4652a4d62..69cad65795f 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/objects/serializer.hpp> #include <vespa/vespalib/objects/deserializer.hpp> +#include <stdexcept> namespace search::expression { diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index b1d3356dd62..4e5bda9cb18 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -108,6 +108,13 @@ public: * @param attribute The attribute vector to use. */ SingleAttributeExecutor(const T & attribute) : _attribute(attribute) { } + void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override { + fef::FeatureExecutor::handle_bind_outputs(outputs_in); + auto o = outputs().get_bound(); + o[1].as_number = 0; // weight + o[2].as_number = 0; // contains + o[3].as_number = 1; // count + } void execute(uint32_t docId) override; }; @@ -120,13 +127,15 @@ private: const T & _attribute; uint32_t _idx; public: - /** - * Constructs an executor. - * - * @param attribute The attribute vector to use. - */ MultiAttributeExecutor(const T & attribute, uint32_t idx) : _attribute(attribute), _idx(idx) { } void execute(uint32_t docId) override; + void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override { + fef::FeatureExecutor::handle_bind_outputs(outputs_in); + auto o = outputs().get_bound(); + o[1].as_number = 0; // weight + o[2].as_number = 0; // contains + o[3].as_number = 0; // count + } }; class CountOnlyAttributeExecutor : public fef::FeatureExecutor { @@ -134,13 +143,15 @@ private: const attribute::IAttributeVector & _attribute; public: - /** - * Constructs an executor. - * - * @param attribute The attribute vector to use. - */ CountOnlyAttributeExecutor(const attribute::IAttributeVector & attribute) : _attribute(attribute) { } void execute(uint32_t docId) override; + void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override { + fef::FeatureExecutor::handle_bind_outputs(outputs_in); + auto o = outputs().get_bound(); + o[0].as_number = 0; // value + o[1].as_number = 0; // weight + o[2].as_number = 0; // contains + } }; /** * Implements the executor for fetching values from a single or array attribute vector @@ -150,8 +161,8 @@ class AttributeExecutor : public fef::FeatureExecutor { private: const attribute::IAttributeVector * _attribute; attribute::BasicType::Type _attrType; - uint32_t _idx; - T _buffer; // used when fetching values from the attribute + uint32_t _idx; + T _buffer; // used when fetching values from the attribute feature_t _defaultCount; public: @@ -163,6 +174,13 @@ public: */ AttributeExecutor(const attribute::IAttributeVector * attribute, uint32_t idx); void execute(uint32_t docId) override; + void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override { + fef::FeatureExecutor::handle_bind_outputs(outputs_in); + auto o = outputs().get_bound(); + o[1].as_number = 0; // weight + o[2].as_number = 0; // contains + o[3].as_number = _defaultCount; // count + } }; @@ -200,9 +218,6 @@ SingleAttributeExecutor<T>::execute(uint32_t docId) o[0].as_number = __builtin_expect(attribute::isUndefined(v), false) ? attribute::getUndefined<feature_t>() : util::getAsFeature(v); - o[1].as_number = 0; // weight - o[2].as_number = 0; // contains - o[3].as_number = 1; // contains } template <typename T> @@ -214,18 +229,12 @@ MultiAttributeExecutor<T>::execute(uint32_t docId) auto o = outputs().get_bound(); o[0].as_number = __builtin_expect(_idx < numValues, true) ? values[_idx].value() : 0; - o[1].as_number = 0; // weight - o[2].as_number = 0; // contains - o[3].as_number = 0; // count } void CountOnlyAttributeExecutor::execute(uint32_t docId) { auto o = outputs().get_bound(); - o[0].as_number = 0; // value - o[1].as_number = 0; // weight - o[2].as_number = 0; // contains o[3].as_number = _attribute.getValueCount(docId); // count } @@ -252,9 +261,6 @@ AttributeExecutor<T>::execute(uint32_t docId) } auto o = outputs().get_bound(); o[0].as_number = value; // value - o[1].as_number = 0; // weight - o[2].as_number = 0; // contains - o[3].as_number = _defaultCount; // count } @@ -356,6 +362,7 @@ createAttributeExecutor(const IAttributeVector *attribute, const vespalib::strin if ((attribute->getCollectionType() == CollectionType::SINGLE) && (attribute->isIntegerType() || attribute->isFloatingPointType())) { { SingleValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator; if (creator.handle(attribute)) return creator.create(stash); } + { SingleValueExecutorCreator<IntegerAttributeTemplate<int8_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } } diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index e1e351d0726..bf8b59f100c 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -67,59 +67,59 @@ namespace search::features { void setup_search_features(fef::IBlueprintRegistry & registry) { // Prod features. - registry.addPrototype(Blueprint::SP(new AgeBlueprint())); - registry.addPrototype(Blueprint::SP(new AttributeBlueprint())); - registry.addPrototype(Blueprint::SP(new AttributeMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new Bm25Blueprint())); - registry.addPrototype(Blueprint::SP(new ClosenessBlueprint())); - registry.addPrototype(Blueprint::SP(new DebugAttributeWaitBlueprint())); - registry.addPrototype(Blueprint::SP(new DebugWaitBlueprint())); - registry.addPrototype(Blueprint::SP(new DistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new DistanceToPathBlueprint())); - registry.addPrototype(Blueprint::SP(new DotProductBlueprint())); - registry.addPrototype(Blueprint::SP(new ElementCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ElementSimilarityBlueprint())); - registry.addPrototype(Blueprint::SP(new EuclideanDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldInfoBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldLengthBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldTermMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new FirstPhaseBlueprint())); - registry.addPrototype(Blueprint::SP(new FlowCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ForeachBlueprint())); - registry.addPrototype(Blueprint::SP(new FreshnessBlueprint())); - registry.addPrototype(Blueprint::SP(new ItemRawScoreBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchCountBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchesBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeAttributeMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeDotProductBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeFieldMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeRankBlueprint())); - registry.addPrototype(Blueprint::SP(new NowBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryTermCountBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomNormalBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomNormalStableBlueprint())); - registry.addPrototype(Blueprint::SP(new RawScoreBlueprint())); - registry.addPrototype(Blueprint::SP(new SubqueriesBlueprint)); - registry.addPrototype(Blueprint::SP(new TensorFromLabelsBlueprint())); - registry.addPrototype(Blueprint::SP(new TensorFromWeightedSetBlueprint())); - registry.addPrototype(Blueprint::SP(new TermBlueprint())); - registry.addPrototype(Blueprint::SP(new TermDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new TermInfoBlueprint())); - registry.addPrototype(Blueprint::SP(new TextSimilarityBlueprint())); - registry.addPrototype(Blueprint::SP(new ValueBlueprint())); + registry.addPrototype(std::make_shared<AgeBlueprint>()); + registry.addPrototype(std::make_shared<AttributeBlueprint>()); + registry.addPrototype(std::make_shared<AttributeMatchBlueprint>()); + registry.addPrototype(std::make_shared<Bm25Blueprint>()); + registry.addPrototype(std::make_shared<ClosenessBlueprint>()); + registry.addPrototype(std::make_shared<DebugAttributeWaitBlueprint>()); + registry.addPrototype(std::make_shared<DebugWaitBlueprint>()); + registry.addPrototype(std::make_shared<DistanceBlueprint>()); + registry.addPrototype(std::make_shared<DistanceToPathBlueprint>()); + registry.addPrototype(std::make_shared<DotProductBlueprint>()); + registry.addPrototype(std::make_shared<ElementCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ElementSimilarityBlueprint>()); + registry.addPrototype(std::make_shared<EuclideanDistanceBlueprint>()); + registry.addPrototype(std::make_shared<FieldInfoBlueprint>()); + registry.addPrototype(std::make_shared<FieldLengthBlueprint>()); + registry.addPrototype(std::make_shared<FieldMatchBlueprint>()); + registry.addPrototype(std::make_shared<FieldTermMatchBlueprint>()); + registry.addPrototype(std::make_shared<FirstPhaseBlueprint>()); + registry.addPrototype(std::make_shared<FlowCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ForeachBlueprint>()); + registry.addPrototype(std::make_shared<FreshnessBlueprint>()); + registry.addPrototype(std::make_shared<ItemRawScoreBlueprint>()); + registry.addPrototype(std::make_shared<MatchBlueprint>()); + registry.addPrototype(std::make_shared<MatchCountBlueprint>()); + registry.addPrototype(std::make_shared<MatchesBlueprint>()); + registry.addPrototype(std::make_shared<NativeAttributeMatchBlueprint>()); + registry.addPrototype(std::make_shared<NativeDotProductBlueprint>()); + registry.addPrototype(std::make_shared<NativeFieldMatchBlueprint>()); + registry.addPrototype(std::make_shared<NativeProximityBlueprint>()); + registry.addPrototype(std::make_shared<NativeRankBlueprint>()); + registry.addPrototype(std::make_shared<NowBlueprint>()); + registry.addPrototype(std::make_shared<QueryBlueprint>()); + registry.addPrototype(std::make_shared<QueryTermCountBlueprint>()); + registry.addPrototype(std::make_shared<RandomBlueprint>()); + registry.addPrototype(std::make_shared<RandomNormalBlueprint>()); + registry.addPrototype(std::make_shared<RandomNormalStableBlueprint>()); + registry.addPrototype(std::make_shared<RawScoreBlueprint>()); + registry.addPrototype(std::make_shared<SubqueriesBlueprint>()); + registry.addPrototype(std::make_shared<TensorFromLabelsBlueprint>()); + registry.addPrototype(std::make_shared<TensorFromWeightedSetBlueprint>()); + registry.addPrototype(std::make_shared<TermBlueprint>()); + registry.addPrototype(std::make_shared<TermDistanceBlueprint>()); + registry.addPrototype(std::make_shared<TermInfoBlueprint>()); + registry.addPrototype(std::make_shared<TextSimilarityBlueprint>()); + registry.addPrototype(std::make_shared<ValueBlueprint>()); // Beta features. - registry.addPrototype(Blueprint::SP(new JaroWinklerDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new ProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ReverseProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new TermEditDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new TermFieldMdBlueprint())); + registry.addPrototype(std::make_shared<JaroWinklerDistanceBlueprint>()); + registry.addPrototype(std::make_shared<ProximityBlueprint>()); + registry.addPrototype(std::make_shared<QueryCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ReverseProximityBlueprint>()); + registry.addPrototype(std::make_shared<TermEditDistanceBlueprint>()); + registry.addPrototype(std::make_shared<TermFieldMdBlueprint>()); registry.addPrototype(std::make_shared<ConstantBlueprint>()); // Ranking Expression diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp index 95813be1360..357aec3fc71 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp @@ -5,8 +5,7 @@ #include <vespa/log/log.h> LOG_SETUP(".fef.blueprintfactory"); -namespace search { -namespace fef { +namespace search::fef { BlueprintFactory::BlueprintFactory() : _blueprintMap() @@ -20,7 +19,7 @@ BlueprintFactory::addPrototype(Blueprint::SP proto) if (_blueprintMap.find(name) != _blueprintMap.end()) { LOG(warning, "Blueprint prototype overwritten: %s", name.c_str()); } - _blueprintMap[name] = proto; + _blueprintMap[name] = std::move(proto); } void @@ -41,9 +40,7 @@ BlueprintFactory::createBlueprint(const vespalib::string &name) const if (itr == _blueprintMap.end()) { return Blueprint::SP(); } - Blueprint::UP bp = itr->second->createInstance(); - return Blueprint::SP(bp.release()); + return itr->second->createInstance(); } -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h index 221cca38f5f..346294ba5f5 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h @@ -6,8 +6,7 @@ #include "iblueprintregistry.h" #include <map> -namespace search { -namespace fef { +namespace search::fef { /** * This class implements the blueprint repository interface and acts @@ -54,6 +53,4 @@ public: Blueprint::SP createBlueprint(const vespalib::string &name) const; }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index a9e500fc802..cd2cd949a91 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -44,13 +44,13 @@ struct Compiler : public Blueprint::DependencyHandler { ExecutorSpec spec; const FeatureNameParser &parser; Frame(Blueprint::SP blueprint, const FeatureNameParser &parser_in) - : spec(blueprint), parser(parser_in) {} + : spec(std::move(blueprint)), parser(parser_in) {} }; using Stack = std::vector<Frame>; struct FrameGuard { Stack &stack; - FrameGuard(Stack &stack_in) : stack(stack_in) {} + explicit FrameGuard(Stack &stack_in) : stack(stack_in) {} ~FrameGuard() { stack.pop_back(); } }; @@ -101,11 +101,11 @@ struct Compiler : public Blueprint::DependencyHandler { FeatureRef setup_feature(const FeatureNameParser &parser, Accept accept_type) { Blueprint::SP blueprint = factory.createBlueprint(parser.baseName()); - if (blueprint.get() == nullptr) { + if ( ! blueprint) { return failed(parser.featureName(), vespalib::make_string("unknown basename: '%s'", parser.baseName().c_str())); } - resolve_stack.emplace_back(blueprint, parser); + resolve_stack.emplace_back(std::move(blueprint), parser); FrameGuard frame_guard(resolve_stack); self().spec.blueprint->setName(parser.executorName()); self().spec.blueprint->attach_dependency_handler(*this); @@ -172,13 +172,13 @@ struct Compiler : public Blueprint::DependencyHandler { } // namespace search::fef::<unnamed> BlueprintResolver::ExecutorSpec::ExecutorSpec(Blueprint::SP blueprint_in) - : blueprint(blueprint_in), + : blueprint(std::move(blueprint_in)), inputs(), output_types() { } -BlueprintResolver::ExecutorSpec::~ExecutorSpec() { } -BlueprintResolver::~BlueprintResolver() { } +BlueprintResolver::ExecutorSpec::~ExecutorSpec() = default; +BlueprintResolver::~BlueprintResolver() = default; BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv) @@ -194,7 +194,7 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, void BlueprintResolver::addSeed(vespalib::stringref feature) { - _seeds.push_back(feature); + _seeds.emplace_back(feature); } bool diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h index 317263260fe..2edca268214 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h @@ -8,8 +8,7 @@ #include "blueprint.h" #include "feature_type.h" -namespace search { -namespace fef { +namespace search::fef { class BlueprintFactory; class IIndexEnvironment; @@ -147,5 +146,4 @@ public: const FeatureMap &getSeedMap() const; }; -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h index 3b7634d3e93..43ca5a44303 100644 --- a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h +++ b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace fef { +namespace search::fef { /** * This is an interface used during plugin setup to register blueprint @@ -23,6 +22,4 @@ public: virtual ~IBlueprintRegistry() {} }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.h b/searchlib/src/vespa/searchlib/fef/parametervalidator.h index 6e0fd7199c8..12044905dd0 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.h +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.h @@ -7,8 +7,7 @@ #include "parameter.h" #include "parameterdescriptions.h" -namespace search { -namespace fef { +namespace search::fef { /** * This class is a validator for a string parameter list given an index environment and a set of parameter descriptions. @@ -83,6 +82,4 @@ public: Result validate(); }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 80729d90bf5..d39be693806 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -153,7 +153,7 @@ RankProgram::resolve(const BlueprintResolver::FeatureMap &features, bool unbox_s } RankProgram::RankProgram(BlueprintResolver::SP resolver) - : _resolver(resolver), + : _resolver(std::move(resolver)), _hot_stash(32768), _cold_stash(), _executors(), @@ -162,7 +162,7 @@ RankProgram::RankProgram(BlueprintResolver::SP resolver) { } -RankProgram::~RankProgram() {} +RankProgram::~RankProgram() = default; void RankProgram::setup(const MatchData &md, diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 440ea3e0ea1..ee88be8ad00 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -27,10 +27,10 @@ using namespace indexproperties; RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv) : _factory(factory), _indexEnv(indexEnv), - _first_phase_resolver(new BlueprintResolver(factory, indexEnv)), - _second_phase_resolver(new BlueprintResolver(factory, indexEnv)), - _summary_resolver(new BlueprintResolver(factory, indexEnv)), - _dumpResolver(new BlueprintResolver(factory, indexEnv)), + _first_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _second_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _summary_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _dumpResolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), _firstPhaseRankFeature(), _secondPhaseRankFeature(), _degradationAttribute(), diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.h b/searchlib/src/vespa/searchlib/index/postinglisthandle.h index 8eb42dc91fe..a9176d7cf13 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.h +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.h @@ -3,6 +3,7 @@ #include <vespa/searchlib/index/postinglistcounts.h> #include <memory> +#include <cstdlib> namespace search { class BitVector; } namespace search::queryeval { class SearchIterator; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp index 164e9cea809..943920c9dc6 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp @@ -53,8 +53,8 @@ QueryTerm::QueryTerm() : QueryTerm::QueryTerm(const QueryTerm &) = default; QueryTerm & QueryTerm::operator = (const QueryTerm &) = default; -QueryTerm::QueryTerm(QueryTerm &&) = default; -QueryTerm & QueryTerm::operator = (QueryTerm &&) = default; +QueryTerm::QueryTerm(QueryTerm &&) noexcept = default; +QueryTerm & QueryTerm::operator = (QueryTerm &&) noexcept = default; QueryTerm::~QueryTerm() = default; diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h index 82ed0eae9a5..65e966ca0f2 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h @@ -57,8 +57,8 @@ public: QueryTerm(std::unique_ptr<QueryNodeResultBase> resultBase, const string & term, const string & index, SearchTerm type); QueryTerm(const QueryTerm &); QueryTerm & operator = (const QueryTerm &); - QueryTerm(QueryTerm &&); - QueryTerm & operator = (QueryTerm &&); + QueryTerm(QueryTerm &&) noexcept; + QueryTerm & operator = (QueryTerm &&) noexcept; ~QueryTerm(); bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp index 2f9242bb3fd..87ab0b57c45 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.cpp @@ -8,21 +8,12 @@ namespace search::queryeval { SearchIteratorPack::~SearchIteratorPack() = default; -SearchIteratorPack::SearchIteratorPack() : _children(), _childMatch(), _md() {} +SearchIteratorPack::SearchIteratorPack() = default; -SearchIteratorPack::SearchIteratorPack(SearchIteratorPack &&rhs) - : _children(std::move(rhs._children)), - _childMatch(std::move(rhs._childMatch)), - _md(std::move(rhs._md)) -{} +SearchIteratorPack::SearchIteratorPack(SearchIteratorPack &&rhs) noexcept = default; SearchIteratorPack & -SearchIteratorPack::operator=(SearchIteratorPack &&rhs) { - _children = std::move(rhs._children); - _childMatch = std::move(rhs._childMatch); - _md = std::move(rhs._md); - return *this; -} +SearchIteratorPack::operator=(SearchIteratorPack &&rhs) noexcept = default; SearchIteratorPack::SearchIteratorPack(const std::vector<SearchIterator*> &children, const std::vector<fef::TermFieldMatchData*> &childMatch, diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h index d35c1749f40..a7b30594a5b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h @@ -20,8 +20,8 @@ private: public: SearchIteratorPack(); ~SearchIteratorPack(); - SearchIteratorPack(SearchIteratorPack &&rhs); - SearchIteratorPack &operator=(SearchIteratorPack &&rhs); + SearchIteratorPack(SearchIteratorPack &&rhs) noexcept; + SearchIteratorPack &operator=(SearchIteratorPack &&rhs) noexcept; SearchIteratorPack(const std::vector<SearchIterator*> &children, const std::vector<fef::TermFieldMatchData*> &childMatch, diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.cpp index c2be2ce37b9..c6801d86351 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.cpp @@ -3,22 +3,18 @@ #include "wand_parts.h" #include <vespa/vespalib/objects/visit.hpp> -namespace search { -namespace queryeval { -namespace wand { +namespace search::queryeval::wand { void VectorizedIteratorTerms::visit_members(vespalib::ObjectVisitor &visitor) const { visit(visitor, "children", _terms); } -VectorizedIteratorTerms::VectorizedIteratorTerms(VectorizedIteratorTerms &&) = default; -VectorizedIteratorTerms & VectorizedIteratorTerms::operator=(VectorizedIteratorTerms &&) = default; -VectorizedIteratorTerms::~VectorizedIteratorTerms() { } +VectorizedIteratorTerms::VectorizedIteratorTerms(VectorizedIteratorTerms &&) noexcept = default; +VectorizedIteratorTerms & VectorizedIteratorTerms::operator=(VectorizedIteratorTerms &&) noexcept = default; +VectorizedIteratorTerms::~VectorizedIteratorTerms() = default; -} // namespace wand -} // namespace queryeval -} // namespace search +} void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const search::queryeval::wand::Term &obj) diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index 9fbf9f7d850..bd60473e05d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -157,8 +157,8 @@ private: public: VectorizedState(); - VectorizedState(VectorizedState &&); - VectorizedState & operator=(VectorizedState &&); + VectorizedState(VectorizedState &&) noexcept; + VectorizedState & operator=(VectorizedState &&) noexcept; ~VectorizedState(); template <typename Scorer, typename Input> @@ -189,14 +189,14 @@ VectorizedState<IteratorPack>::VectorizedState() _iteratorPack() {} template <typename IteratorPack> -VectorizedState<IteratorPack>::~VectorizedState() { } +VectorizedState<IteratorPack>::~VectorizedState() = default; template <typename IteratorPack> -VectorizedState<IteratorPack>::VectorizedState(VectorizedState &&) = default; +VectorizedState<IteratorPack>::VectorizedState(VectorizedState &&) noexcept = default; template <typename IteratorPack> VectorizedState<IteratorPack> & -VectorizedState<IteratorPack>::operator=(VectorizedState &&) = default; +VectorizedState<IteratorPack>::operator=(VectorizedState &&) noexcept = default; template <typename IteratorPack> template <typename Scorer, typename Input> @@ -239,8 +239,8 @@ public: template <typename Scorer> VectorizedIteratorTerms(const Terms &t, const Scorer &, uint32_t docIdLimit, fef::MatchData::UP childrenMatchData); - VectorizedIteratorTerms(VectorizedIteratorTerms &&); - VectorizedIteratorTerms & operator=(VectorizedIteratorTerms &&); + VectorizedIteratorTerms(VectorizedIteratorTerms &&) noexcept; + VectorizedIteratorTerms & operator=(VectorizedIteratorTerms &&) noexcept; ~VectorizedIteratorTerms(); void unpack(uint16_t ref, uint32_t docid) { iteratorPack().unpack(ref, docid); } diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index ea5d30d9a47..9175168248c 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_library(searchlib_tensor OBJECT dense_tensor_store.cpp generic_tensor_attribute.cpp generic_tensor_store.cpp + hnsw_index.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp tensor_attribute.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h new file mode 100644 index 00000000000..8dfb77ddccb --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::tensor { struct TypedCells; } + +namespace search::tensor { + +/** + * Interface used to calculate the distance between two n-dimensional vectors. + * + * The vectors must be of same size and same type (float or double). + * The actual implementation must know which type the vectors are. + */ +class DistanceFunction { +public: + virtual ~DistanceFunction() {} + virtual double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h new file mode 100644 index 00000000000..1e8727e92aa --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -0,0 +1,34 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "distance_function.h" + +namespace search::tensor { + +// TODO: Make distance functions hardware optimized. + +/** + * Calculates the square of the standard Euclidean distance. + */ +template <typename FloatType> +class SquaredEuclideanDistance : public DistanceFunction { +public: + double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { + auto lhs_vector = lhs.typify<FloatType>(); + auto rhs_vector = rhs.typify<FloatType>(); + double result = 0.0; + size_t sz = lhs_vector.size(); + assert(sz == rhs_vector.size()); + for (size_t i = 0; i < sz; ++i) { + double diff = lhs_vector[i] - rhs_vector[i]; + result += diff * diff; + } + return result; + } +}; + +template class SquaredEuclideanDistance<float>; +template class SquaredEuclideanDistance<double>; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h new file mode 100644 index 00000000000..e5dfa35a529 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <cstdint> + +namespace search::tensor { + +/** + * Interface that provides access to the vector that is associated with the the given document id. + * + * All vectors should be the same size and either of type float or double. + */ +class DocVectorAccess { +public: + virtual ~DocVectorAccess() {} + virtual vespalib::tensor::TypedCells get_vector(uint32_t docid) const = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp index 4e522f27ce2..ff41396f66b 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp @@ -3,7 +3,6 @@ #include "generic_tensor_store.h" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/serializable.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/objects/nbostream.h> @@ -15,9 +14,7 @@ using search::datastore::Handle; using vespalib::tensor::Tensor; using vespalib::tensor::TypedBinaryFormat; -namespace search { - -namespace tensor { +namespace search::tensor { constexpr size_t MIN_BUFFER_ARRAYS = 1024; @@ -118,6 +115,4 @@ GenericTensorStore::setTensor(const Tensor &tensor) return raw.ref; } -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp new file mode 100644 index 00000000000..be53b758841 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -0,0 +1,367 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distance_function.h" +#include "hnsw_index.h" +#include "random_level_generator.h" +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/util/rcuvector.hpp> + +namespace search::tensor { + +namespace { + +// TODO: Move this to MemoryAllocator, with name PAGE_SIZE. +constexpr size_t small_page_size = 4 * 1024; +constexpr size_t min_num_arrays_for_new_buffer = 8 * 1024; +constexpr float alloc_grow_factor = 0.2; +// TODO: Adjust these numbers to what we accept as max in config. +constexpr size_t max_level_array_size = 16; +constexpr size_t max_link_array_size = 64; + +} + +search::datastore::ArrayStoreConfig +HnswIndex::make_default_node_store_config() +{ + return NodeStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true); +} + +search::datastore::ArrayStoreConfig +HnswIndex::make_default_link_store_config() +{ + return LinkStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true); +} + +uint32_t +HnswIndex::max_links_for_level(uint32_t level) const +{ + return (level == 0) ? _cfg.max_links_at_level_0() : _cfg.max_links_at_hierarchic_levels(); +} + +uint32_t +HnswIndex::make_node_for_document(uint32_t docid) +{ + uint32_t max_level = _level_generator.max_level(); + // TODO: Add capping on num_levels + uint32_t num_levels = max_level + 1; + // Note: The level array instance lives as long as the document is present in the index. + LevelArray levels(num_levels, AtomicEntryRef()); + auto node_ref = _nodes.add(levels); + _node_refs[docid].store_release(node_ref); + return max_level; +} + +HnswIndex::LevelArrayRef +HnswIndex::get_level_array(uint32_t docid) const +{ + auto node_ref = _node_refs[docid].load_acquire(); + return _nodes.get(node_ref); +} + +HnswIndex::LinkArrayRef +HnswIndex::get_link_array(uint32_t docid, uint32_t level) const +{ + auto levels = get_level_array(docid); + assert(level < levels.size()); + return _links.get(levels[level].load_acquire()); +} + +void +HnswIndex::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links) +{ + auto links_ref = _links.add(links); + auto node_ref = _node_refs[docid].load_acquire(); + auto levels = _nodes.get_writable(node_ref); + levels[level].store_release(links_ref); +} + +bool +HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArray& result) const +{ + for (uint32_t result_docid : result) { + double dist = calc_distance(candidate.docid, result_docid); + if (dist < candidate.distance) { + return true; + } + } + return false; +} + +HnswIndex::LinkArray +HnswIndex::select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + HnswCandidateVector sorted(neighbors); + std::sort(sorted.begin(), sorted.end(), LesserDistance()); + LinkArray result; + for (size_t i = 0, m = std::min(static_cast<size_t>(max_links), sorted.size()); i < m; ++i) { + result.push_back(sorted[i].docid); + } + return result; +} + +HnswIndex::LinkArray +HnswIndex::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + LinkArray result; + bool need_filtering = neighbors.size() > max_links; + NearestPriQ nearest; + for (const auto& entry : neighbors) { + nearest.push(entry); + } + while (!nearest.empty()) { + auto candidate = nearest.top(); + nearest.pop(); + if (need_filtering && have_closer_distance(candidate, result)) { + continue; + } + result.push_back(candidate.docid); + if (result.size() == max_links) { + return result; + } + } + return result; +} + +HnswIndex::LinkArray +HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + if (_cfg.heuristic_select_neighbors()) { + return select_neighbors_heuristic(neighbors, max_links); + } else { + return select_neighbors_simple(neighbors, max_links); + } +} + +void +HnswIndex::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level) +{ + set_link_array(docid, level, neighbors); + for (uint32_t neighbor_docid : neighbors) { + auto old_links = get_link_array(neighbor_docid, level); + LinkArray new_links(old_links.begin(), old_links.end()); + new_links.push_back(docid); + set_link_array(neighbor_docid, level, new_links); + } +} + +void +HnswIndex::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level) +{ + LinkArray new_links; + auto old_links = get_link_array(remove_from, level); + for (uint32_t id : old_links) { + if (id != remove_id) new_links.push_back(id); + } + set_link_array(remove_from, level, new_links); +} + + +double +HnswIndex::calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const +{ + auto lhs = get_vector(lhs_docid); + return calc_distance(lhs, rhs_docid); +} + +double +HnswIndex::calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const +{ + auto rhs = get_vector(rhs_docid); + return _distance_func.calc(lhs, rhs); +} + +HnswCandidate +HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) +{ + HnswCandidate nearest = entry_point; + bool keep_searching = true; + while (keep_searching) { + keep_searching = false; + for (uint32_t neighbor_docid : get_link_array(nearest.docid, level)) { + double dist = calc_distance(input, neighbor_docid); + if (dist < nearest.distance) { + nearest = HnswCandidate(neighbor_docid, dist); + keep_searching = true; + } + } + } + return nearest; +} + +void +HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) +{ + NearestPriQ candidates; + // TODO: Add proper handling of visited set. + auto visited = BitVector::create(_node_refs.size()); + for (const auto &entry : best_neighbors.peek()) { + candidates.push(entry); + visited->setBit(entry.docid); + } + double limit_dist = std::numeric_limits<double>::max(); + + while (!candidates.empty()) { + auto cand = candidates.top(); + if (cand.distance > limit_dist) { + break; + } + candidates.pop(); + for (uint32_t neighbor_docid : get_link_array(cand.docid, level)) { + if (visited->testBit(neighbor_docid)) { + continue; + } + visited->setBit(neighbor_docid); + double dist_to_input = calc_distance(input, neighbor_docid); + if (dist_to_input < limit_dist) { + candidates.emplace(neighbor_docid, dist_to_input); + best_neighbors.emplace(neighbor_docid, dist_to_input); + if (best_neighbors.size() > neighbors_to_find) { + best_neighbors.pop(); + limit_dist = best_neighbors.top().distance; + } + } + } + } +} + +HnswIndex::HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, + RandomLevelGenerator& level_generator, const Config& cfg) + : _vectors(vectors), + _distance_func(distance_func), + _level_generator(level_generator), + _cfg(cfg), + _node_refs(), + _nodes(make_default_node_store_config()), + _links(make_default_link_store_config()), + _entry_docid(0), // Note that docid 0 is reserved and never used + _entry_level(-1) +{ +} + +HnswIndex::~HnswIndex() = default; + +void +HnswIndex::add_document(uint32_t docid) +{ + auto input = get_vector(docid); + _node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + int level = make_node_for_document(docid); + if (_entry_docid == 0) { + _entry_docid = docid; + _entry_level = level; + return; + } + + int search_level = _entry_level; + double entry_dist = calc_distance(input, _entry_docid); + HnswCandidate entry_point(_entry_docid, entry_dist); + while (search_level > level) { + entry_point = find_nearest_in_layer(input, entry_point, search_level); + --search_level; + } + + FurthestPriQ best_neighbors; + best_neighbors.push(entry_point); + search_level = std::min(level, _entry_level); + + // Insert the added document in each level it should exist in. + while (search_level >= 0) { + // TODO: Rename to search_level? + search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); + auto neighbors = select_neighbors(best_neighbors.peek(), max_links_for_level(search_level)); + connect_new_node(docid, neighbors, search_level); + // TODO: Shrink neighbors if needed + --search_level; + } + if (level > _entry_level) { + _entry_docid = docid; + _entry_level = level; + } +} + +void +HnswIndex::remove_document(uint32_t docid) +{ + bool need_new_entrypoint = (docid == _entry_docid); + LinkArray empty; + LevelArrayRef node_levels = get_level_array(docid); + for (int level = node_levels.size(); level-- > 0; ) { + LinkArrayRef my_links = get_link_array(docid, level); + for (uint32_t neighbor_id : my_links) { + if (need_new_entrypoint) { + _entry_docid = neighbor_id; + _entry_level = level; + need_new_entrypoint = false; + } + remove_link_to(neighbor_id, docid, level); + } + set_link_array(docid, level, empty); + } + if (need_new_entrypoint) { + _entry_docid = 0; + _entry_level = -1; + } + search::datastore::EntryRef invalid; + _node_refs[docid].store_release(invalid); +} + +std::vector<uint32_t> +HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) +{ + std::vector<uint32_t> result; + FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k)); + while (candidates.size() > k) { + candidates.pop(); + } + result.reserve(candidates.size()); + for (const HnswCandidate & hit : candidates.peek()) { + result.emplace_back(hit.docid); + } + std::sort(result.begin(), result.end()); + return result; +} + +FurthestPriQ +HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) +{ + FurthestPriQ best_neighbors; + if (_entry_level < 0) { + return best_neighbors; + } + double entry_dist = calc_distance(vector, _entry_docid); + HnswCandidate entry_point(_entry_docid, entry_dist); + int search_level = _entry_level; + while (search_level > 0) { + entry_point = find_nearest_in_layer(vector, entry_point, search_level); + --search_level; + } + best_neighbors.push(entry_point); + search_layer(vector, k, best_neighbors, 0); + return best_neighbors; +} + +HnswNode +HnswIndex::get_node(uint32_t docid) const +{ + auto node_ref = _node_refs[docid].load_acquire(); + if (!node_ref.valid()) { + return HnswNode(); + } + auto levels = _nodes.get(node_ref); + HnswNode::LevelArray result; + for (const auto& links_ref : levels) { + auto links = _links.get(links_ref.load_acquire()); + HnswNode::LinkArray result_links(links.begin(), links.end()); + std::sort(result_links.begin(), result_links.end()); + result.push_back(result_links); + } + return HnswNode(result); +} + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h new file mode 100644 index 00000000000..814148072ca --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -0,0 +1,152 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "doc_vector_access.h" +#include "hnsw_index_utils.h" +#include "hnsw_node.h" +#include "nearest_neighbor_index.h" +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/datastore/array_store.h> +#include <vespa/vespalib/datastore/atomic_entry_ref.h> +#include <vespa/vespalib/datastore/entryref.h> +#include <vespa/vespalib/util/rcuvector.h> + +namespace search::tensor { + +class DistanceFunction; +class RandomLevelGenerator; + +/** + * Implementation of a hierarchical navigable small world graph (HNSW) + * that is used for approximate K-nearest neighbor search. + * + * The implementation supports 1 write thread and multiple search threads without the use of mutexes. + * This is achieved by using data stores that use generation tracking and associated memory management. + * + * The implementation is mainly based on the algorithms described in + * "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs" (Yu. A. Malkov, D. A. Yashunin), + * but some adjustments are made to support proper removes. + * + * TODO: Add details on how to handle removes. + */ +class HnswIndex : public NearestNeighborIndex { +public: + class Config { + private: + uint32_t _max_links_at_level_0; + uint32_t _max_links_at_hierarchic_levels; + uint32_t _neighbors_to_explore_at_construction; + bool _heuristic_select_neighbors; + + public: + Config(uint32_t max_links_at_level_0_in, + uint32_t max_links_at_hierarchic_levels_in, + uint32_t neighbors_to_explore_at_construction_in, + bool heuristic_select_neighbors_in) + : _max_links_at_level_0(max_links_at_level_0_in), + _max_links_at_hierarchic_levels(max_links_at_hierarchic_levels_in), + _neighbors_to_explore_at_construction(neighbors_to_explore_at_construction_in), + _heuristic_select_neighbors(heuristic_select_neighbors_in) + {} + uint32_t max_links_at_level_0() const { return _max_links_at_level_0; } + uint32_t max_links_at_hierarchic_levels() const { return _max_links_at_hierarchic_levels; } + uint32_t neighbors_to_explore_at_construction() const { return _neighbors_to_explore_at_construction; } + bool heuristic_select_neighbors() const { return _heuristic_select_neighbors; } + }; + +protected: + using AtomicEntryRef = search::datastore::AtomicEntryRef; + + // This uses 10 bits for buffer id -> 1024 buffers. + // As we have very short arrays we get less fragmentation with fewer and larger buffers. + using EntryRefType = search::datastore::EntryRefT<22>; + + // Provides mapping from document id -> node reference. + // The reference is used to lookup the node data in NodeStore. + using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>; + + // This stores the level arrays for all nodes. + // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. + using NodeStore = search::datastore::ArrayStore<AtomicEntryRef, EntryRefType>; + using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArray = vespalib::Array<AtomicEntryRef>; + + // This stores all link arrays. + // A link array consists of the document ids of the nodes a particular node is linked to. + using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>; + using LinkArrayRef = LinkStore::ConstArrayRef; + using LinkArray = vespalib::Array<uint32_t>; + + using TypedCells = vespalib::tensor::TypedCells; + + const DocVectorAccess& _vectors; + const DistanceFunction& _distance_func; + RandomLevelGenerator& _level_generator; + Config _cfg; + NodeRefVector _node_refs; + NodeStore _nodes; + LinkStore _links; + uint32_t _entry_docid; + int _entry_level; + + static search::datastore::ArrayStoreConfig make_default_node_store_config(); + static search::datastore::ArrayStoreConfig make_default_link_store_config(); + + uint32_t max_links_for_level(uint32_t level) const; + uint32_t make_node_for_document(uint32_t docid); + LevelArrayRef get_level_array(uint32_t docid) const; + LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; + void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); + + /** + * Returns true if the distance between the candidate and a node in the current result + * is less than the distance between the candidate and the node we want to add to the graph. + * In this case the candidate should be discarded as we already are connected to the space + * where the candidate is located. + * Used by select_neighbors_heuristic(). + */ + bool have_closer_distance(HnswCandidate candidate, const LinkArray& curr_result) const; + LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; + LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; + LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; + void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level); + void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); + + inline TypedCells get_vector(uint32_t docid) const { + return _vectors.get_vector(docid); + } + + double calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const; + double calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const; + + /** + * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. + */ + HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level); + void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level); + +public: + HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func, + RandomLevelGenerator& level_generator, const Config& cfg); + ~HnswIndex() override; + + void add_document(uint32_t docid) override; + void remove_document(uint32_t docid) override; + std::vector<uint32_t> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) override; + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k); + + // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) + + uint32_t get_entry_docid() const { return _entry_docid; } + uint32_t get_entry_level() const { return _entry_level; } + + // Should only be used by unit tests. + HnswNode get_node(uint32_t docid) const; + + // TODO: Implement set_node() as well for use in unit tests. +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h new file mode 100644 index 00000000000..b11d3f36a7a --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -0,0 +1,48 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <queue> +#include <vector> + +namespace search::tensor { + +/** + * Represents a candidate node with its distance to another point in space. + */ +struct HnswCandidate { + uint32_t docid; + double distance; + HnswCandidate(uint32_t docid_in, double distance_in) : docid(docid_in), distance(distance_in) {} +}; + +struct GreaterDistance { + bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const { + return (rhs.distance < lhs.distance); + } +}; + +struct LesserDistance { + bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const { + return (lhs.distance < rhs.distance); + } +}; + +using HnswCandidateVector = std::vector<HnswCandidate>; + +/** + * Priority queue that keeps the candidate node that is nearest a point in space on top. + */ +using NearestPriQ = std::priority_queue<HnswCandidate, HnswCandidateVector, GreaterDistance>; + +/** + * Priority queue that keeps the candidate node that is furthest away a point in space on top. + */ +class FurthestPriQ : public std::priority_queue<HnswCandidate, HnswCandidateVector, LesserDistance> { +public: + const HnswCandidateVector& peek() const { return c; } +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h new file mode 100644 index 00000000000..8a6187ffbcc --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vector> + +namespace search::tensor { + +/** + * Represents a snapshot of a graph node with all its levels and links. + * Should only be used by unit tests. + */ +class HnswNode { +public: + using LinkArray = std::vector<uint32_t>; + using LevelArray = std::vector<LinkArray>; + +private: + LevelArray _levels; + +public: + HnswNode() : _levels() {} + HnswNode(const LinkArray& level_0) : _levels() { _levels.push_back(level_0); } + HnswNode(const LevelArray& levels_in) : _levels(levels_in) {} + bool empty() const { return _levels.empty(); } + size_t size() const { return _levels.size(); } + const LevelArray& levels() const { return _levels; } + const LinkArray& level(size_t idx) const { return _levels[idx]; } + bool operator==(const HnswNode& rhs) { + return _levels == rhs._levels; + } +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h new file mode 100644 index 00000000000..2ae322fe76e --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -0,0 +1,22 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <vector> +#include <vespa/eval/tensor/dense/typed_cells.h> + +namespace search::tensor { + +/** + * Interface for an index that is used for (approximate) nearest neighbor search. + */ +class NearestNeighborIndex { +public: + virtual ~NearestNeighborIndex() {} + virtual void add_document(uint32_t docid) = 0; + virtual void remove_document(uint32_t docid) = 0; + virtual std::vector<uint32_t> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h new file mode 100644 index 00000000000..0fcac977d9d --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h @@ -0,0 +1,16 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::tensor { + +/** + * Interface used to randomly draw the max level a new hnsw node should exist in. + */ +class RandomLevelGenerator { +public: + virtual ~RandomLevelGenerator() {} + virtual uint32_t max_level() = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index 0de200ae32c..04aa790e4cb 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -1282,7 +1282,7 @@ FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>::doSeek(uint32_t docId) s1CacheInt, K_VALUE_FILTEROCC_L1SKIPDELTA_BITPOS, EC, _l1SkipDocIdBitsOffset += 1 +); - assert(docIdBitsOffset = _l1SkipDocIdBitsOffset); + assert(docIdBitsOffset == _l1SkipDocIdBitsOffset); if (__builtin_expect(oDocId >= _l2SkipDocId, false)) { // Validate L2 Skip information assert(oDocId == _l2SkipDocId); diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp index 5ef9b6cb2d4..4c80c6f869c 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp @@ -61,4 +61,9 @@ MockAttributeManager::addAttribute(const AttributeVector::SP &attr) { addAttribute(attr->getName(), attr); } +std::shared_ptr<attribute::ReadableAttributeVector> +MockAttributeManager::readable_attribute_vector(const string& name) const { + return findAttribute(name); +} + } diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h index dbf84a40e84..a8ec686433a 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h @@ -28,6 +28,7 @@ public: IAttributeContext::UP createContext() const override; void addAttribute(const vespalib::string &name, const AttributeVector::SP &attr); void addAttribute(const AttributeVector::SP &attr); + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index a84e27b2e53..a5eaa61af12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -37,7 +37,7 @@ Packet::Packet(const void * buf, size_t sz) : _limit(sz), _buf(static_cast<const char *>(buf), sz) { - nbostream_longlivedbuf os(_buf.c_str(), sz); + nbostream_longlivedbuf os(_buf.data(), sz); while ( os.size() > 0 ) { Entry e; e.deserialize(os); @@ -55,7 +55,7 @@ bool Packet::merge(const Packet & packet) if (retval) { _count += packet._count; _range.to(packet._range.to()); - _buf.write(packet.getHandle().c_str(), packet.getHandle().size()); + _buf.write(packet.getHandle().data(), packet.getHandle().size()); } return retval; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index fc9518ccf1b..5a64d829183 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -282,7 +282,7 @@ void waitPendingSync(vespalib::Monitor &syncMonitor, bool &pendingSync) void Domain::commit(const Packet & packet) { DomainPart::SP dp(_parts.rbegin()->second); - vespalib::nbostream_longlivedbuf is(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); if (dp->byteSize() > _domainPartSize) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index d2838711a51..8a6e833bd1f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -403,7 +403,7 @@ void DomainPart::commit(SerialNum firstSerial, const Packet &packet) { int64_t firstPos(_transLog->GetPosition()); - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); if (_range.from() == 0) { _range.from(firstSerial); } @@ -495,7 +495,7 @@ DomainPart::visit(SerialNumRange &r, Packet &packet) } } else { const nbostream & tmp = start->second.getHandle(); - nbostream_longlivedbuf h(tmp.c_str(), tmp.size()); + nbostream_longlivedbuf h(tmp.data(), tmp.size()); LOG(debug, "Visit partial[%" PRIu64 ", %" PRIu64 "] (%zd, %zd, %zd)", start->second.range().from(), start->second.range().to(), h.rp(), h.size(), h.capacity()); Packet newPacket(h.size()); @@ -585,13 +585,13 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) size_t start(os.size()); entry.serialize(os); size_t end(os.size()); - crc = calcCrc(_defaultCrc, os.c_str()+start, end - start); + crc = calcCrc(_defaultCrc, os.data() + start, end - start); os << crc; size_t osSize = os.size(); assert(osSize == len + sizeof(len) + sizeof(uint8_t)); LockGuard guard(_writeLock); - if ( ! file.CheckedWrite(os.c_str(), osSize) ) { + if ( ! file.CheckedWrite(os.data(), osSize) ) { throw runtime_error(handleWriteError("Failed writing the entry.", file, lastKnownGoodPos, entry, end - start)); } _writtenSerial = entry.serial(); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 37903bc21f5..a3528c4f615 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -366,7 +366,7 @@ public: req->SetMethodName("visitCallback"); req->GetParams()->AddString(domain.c_str()); req->GetParams()->AddInt32(id); - req->GetParams()->AddData(packet.getHandle().c_str(), packet.getHandle().size()); + req->GetParams()->AddData(packet.getHandle().data(), packet.getHandle().size()); return send(req); } diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index 559042a8c7a..e38763f9ba8 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -8,6 +8,7 @@ #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> +#include <stdexcept> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.fileutil"); diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 496a19d153f..2e1a808c493 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -2,6 +2,7 @@ #include "url.h" #include <algorithm> +#include <cstdio> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.url"); diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index b161958fc43..4cad98a8e01 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -100,6 +100,10 @@ public: IAttributeContext::UP createContext() const override { return IAttributeContext::UP(new MyAttributeContext(_attr)); } + + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override { + LOG_ABORT("should not be reached"); + } }; struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index a6a5263f5ac..030cf7e82c1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -3,6 +3,7 @@ #include "general_result.h" #include "resultconfig.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/datatype.h> #include <zlib.h> #include <cassert> @@ -96,7 +97,13 @@ std::unique_ptr<document::FieldValue> GeneralResult::get_field_value(const vespalib::string& field_name) const { if (_document != nullptr) { - return _document->getValue(field_name); + const document::Field & field = _document->getField(field_name); + auto value(field.getDataType().createFieldValue()); + if (value) { + if (_document->getValue(field, *value)) { + return value; + } + } } return std::unique_ptr<document::FieldValue>(); } diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 97b6cc344e1..cefa8ab2f51 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -19,11 +19,16 @@ import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.security.GeneralSecurityException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Random; import static com.yahoo.security.Extension.SUBJECT_ALTERNATIVE_NAMES; import static java.util.stream.Collectors.toList; @@ -140,4 +145,20 @@ public class X509CertificateUtils { } } + public static boolean privateKeyMatchesPublicKey(PrivateKey privateKey, PublicKey publicKey) { + byte[] someRandomData = new byte[64]; + new Random().nextBytes(someRandomData); + + Signature signer = SignatureUtils.createSigner(privateKey); + Signature verifier = SignatureUtils.createVerifier(publicKey); + try { + signer.update(someRandomData); + verifier.update(someRandomData); + byte[] signature = signer.sign(); + return verifier.verify(signature); + } catch (SignatureException e) { + throw new RuntimeException(e); + } + } + } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index e12ea3cf47d..886cf3e886b 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -35,7 +35,7 @@ public interface TlsContext extends AutoCloseable { "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3, Java 12 Set<String> ALLOWED_PROTOCOLS = com.yahoo.vespa.jdk8compat.Set.of("TLSv1.2"); // TODO Enable TLSv1.3 - String SSL_CONTEXT_VERSION = "TLSv1.2"; // TODO Enable TLSv1.3 + String SSL_CONTEXT_VERSION = "TLS"; // Use SSLContext implementations that supports all TLS versions /** * @return the allowed cipher suites supported by the provided context instance diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java index c0e9e1053c3..5db6d551193 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java @@ -30,6 +30,7 @@ public class TransportSecurityOptions { private final Path caCertificatesFile; private final AuthorizedPeers authorizedPeers; private final List<String> acceptedCiphers; + private final boolean isHostnameValidationDisabled; private TransportSecurityOptions(Builder builder) { this.privateKeyFile = builder.privateKeyFile; @@ -37,6 +38,7 @@ public class TransportSecurityOptions { this.caCertificatesFile = builder.caCertificatesFile; this.authorizedPeers = builder.authorizedPeers; this.acceptedCiphers = builder.acceptedCiphers; + this.isHostnameValidationDisabled = builder.isHostnameValidationDisabled; } public Optional<Path> getPrivateKeyFile() { @@ -57,6 +59,8 @@ public class TransportSecurityOptions { public List<String> getAcceptedCiphers() { return acceptedCiphers; } + public boolean isHostnameValidationDisabled() { return isHostnameValidationDisabled; } + public static TransportSecurityOptions fromJsonFile(Path file) { try (InputStream in = Files.newInputStream(file)) { return new TransportSecurityOptionsJsonSerializer().deserialize(in); @@ -90,6 +94,7 @@ public class TransportSecurityOptions { private Path caCertificatesFile; private AuthorizedPeers authorizedPeers; private List<String> acceptedCiphers = new ArrayList<>(); + private boolean isHostnameValidationDisabled; public Builder() {} @@ -114,6 +119,11 @@ public class TransportSecurityOptions { return this; } + public Builder withHostnameValidationDisabled(boolean isDisabled) { + this.isHostnameValidationDisabled = isDisabled; + return this; + } + public TransportSecurityOptions build() { return new TransportSecurityOptions(this); } @@ -127,6 +137,7 @@ public class TransportSecurityOptions { ", caCertificatesFile=" + caCertificatesFile + ", authorizedPeers=" + authorizedPeers + ", acceptedCiphers=" + acceptedCiphers + + ", isHostnameValidationDisabled=" + isHostnameValidationDisabled + '}'; } @@ -135,7 +146,8 @@ public class TransportSecurityOptions { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TransportSecurityOptions that = (TransportSecurityOptions) o; - return Objects.equals(privateKeyFile, that.privateKeyFile) && + return isHostnameValidationDisabled == that.isHostnameValidationDisabled && + Objects.equals(privateKeyFile, that.privateKeyFile) && Objects.equals(certificatesFile, that.certificatesFile) && Objects.equals(caCertificatesFile, that.caCertificatesFile) && Objects.equals(authorizedPeers, that.authorizedPeers) && @@ -144,6 +156,6 @@ public class TransportSecurityOptions { @Override public int hashCode() { - return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers); + return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers, isHostnameValidationDisabled); } }
\ No newline at end of file diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java index 6594fa84255..2b001ca2ca0 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; /** * Jackson bindings for transport security options @@ -20,6 +21,7 @@ class TransportSecurityOptionsEntity { @JsonProperty("files") Files files; @JsonProperty("authorized-peers") @JsonInclude(NON_EMPTY) List<AuthorizedPeer> authorizedPeers; @JsonProperty("accepted-ciphers") @JsonInclude(NON_EMPTY) List<String> acceptedCiphers; + @JsonProperty("disable-hostname-validation") @JsonInclude(NON_NULL) Boolean isHostnameValidationDisabled; static class Files { @JsonProperty("private-key") String privateKeyFile; diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java index 5487bad24e7..3cba434912c 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java @@ -77,6 +77,9 @@ public class TransportSecurityOptionsJsonSerializer { } builder.withAcceptedCiphers(entity.acceptedCiphers); } + if (entity.isHostnameValidationDisabled != null) { + builder.withHostnameValidationDisabled(entity.isHostnameValidationDisabled); + } return builder.build(); } @@ -158,6 +161,9 @@ public class TransportSecurityOptionsJsonSerializer { if (!options.getAcceptedCiphers().isEmpty()) { entity.acceptedCiphers = options.getAcceptedCiphers(); } + if (options.isHostnameValidationDisabled()) { + entity.isHostnameValidationDisabled = true; + } return entity; } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 76a93028efe..b4eca8328c1 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -17,7 +17,9 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -71,4 +73,18 @@ public class X509CertificateUtilsTest { assertThat(sans.size(), is(1)); assertThat(sans.get(0), equalTo(san)); } + + @Test + public void verifies_matching_cert_and_key() { + KeyPair ecKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair ecKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair rsaKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + KeyPair rsaKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairA.getPublic())); + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairA.getPublic())); + + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairB.getPublic())); + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairB.getPublic())); + } }
\ No newline at end of file diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java index 28dc10d31d5..f2d2b932cd0 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java @@ -21,6 +21,7 @@ public class TransportSecurityOptionsTest { .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withHostnameValidationDisabled(true) .build(); @Test diff --git a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java index 078aa58c948..0dec75fa711 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java @@ -22,7 +22,6 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.CN; import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.SAN_DNS; @@ -44,6 +43,7 @@ public class TransportSecurityOptionsJsonSerializerTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCaCertificates(Paths.get("/path/to/ca-certs.pem")) .withCertificates(Paths.get("/path/to/cert.pem"), Paths.get("/path/to/key.pem")) + .withHostnameValidationDisabled(false) .withAuthorizedPeers( new AuthorizedPeers( new HashSet<>(Arrays.asList( @@ -66,6 +66,7 @@ public class TransportSecurityOptionsJsonSerializerTest { .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withHostnameValidationDisabled(true) .build(); File outputFile = tempDirectory.newFile(); try (OutputStream out = Files.newOutputStream(outputFile.toPath())) { @@ -76,4 +77,22 @@ public class TransportSecurityOptionsJsonSerializerTest { assertJsonEquals(expectedOutput, actualOutput); } + @Test + public void disable_hostname_validation_is_not_serialized_if_false() throws IOException { + TransportSecurityOptions options = new TransportSecurityOptions.Builder() + .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) + .withCaCertificates(Paths.get("my_cas.pem")) + .withHostnameValidationDisabled(false) + .build(); + File outputFile = tempDirectory.newFile(); + try (OutputStream out = Files.newOutputStream(outputFile.toPath())) { + new TransportSecurityOptionsJsonSerializer().serialize(out, options); + } + + String expectedOutput = new String(Files.readAllBytes( + Paths.get("src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json"))); + String actualOutput = new String(Files.readAllBytes(outputFile.toPath())); + assertJsonEquals(expectedOutput, actualOutput); + } + } diff --git a/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json new file mode 100644 index 00000000000..0506c130722 --- /dev/null +++ b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json @@ -0,0 +1,7 @@ +{ + "files": { + "private-key": "myhost.key", + "ca-certificates": "my_cas.pem", + "certificates": "certs.pem" + } +}
\ No newline at end of file diff --git a/security-utils/src/test/resources/transport-security-options.json b/security-utils/src/test/resources/transport-security-options.json index 2e55c8fd931..7983982f644 100644 --- a/security-utils/src/test/resources/transport-security-options.json +++ b/security-utils/src/test/resources/transport-security-options.json @@ -1,8 +1,9 @@ { + "disable-hostname-validation": true, "files": { "private-key": "myhost.key", "ca-certificates": "my_cas.pem", "certificates": "certs.pem" - }, - "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"] + }, + "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"] }
\ No newline at end of file diff --git a/staging_vespalib/src/tests/objects/identifiable_test.cpp b/staging_vespalib/src/tests/objects/identifiable_test.cpp index b952ad18333..2b4d50cc786 100644 --- a/staging_vespalib/src/tests/objects/identifiable_test.cpp +++ b/staging_vespalib/src/tests/objects/identifiable_test.cpp @@ -161,40 +161,40 @@ void IdentifiableTest::testNboStream() EXPECT_EQUAL(nbostream::ok, s.state()); EXPECT_EQUAL(10u, s.size()); EXPECT_EQUAL(16u, s.capacity()); - EXPECT_EQUAL(0, strncmp(s.c_str()+4, "abcdef", 6)); + EXPECT_EQUAL(0, strncmp(s.data() + 4, "abcdef", 6)); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s.reserve(50); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - prev = s.c_str(); + prev = s.data(); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); } { nbostream s; diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index 6cc2af1fc90..2465d5f9d9b 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -244,7 +244,7 @@ int Identifiable::onCmp(const Identifiable& b) const nbs << b; size_t minLength(std::min(as.size(), bs.size())); if (minLength > 0) { - diff = memcmp(as.c_str(), bs.c_str(), minLength); + diff = memcmp(as.data(), bs.data(), minLength); } if (diff == 0) { diff = as.size() - bs.size(); diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h index 4a87dde82e4..e452c35cabe 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h @@ -190,12 +190,12 @@ public: RuntimeInfo * _rt; }; DECLARE_IDENTIFIABLE_ROOT(Identifiable); - Identifiable() { } - Identifiable(Identifiable &&) = default; - Identifiable & operator = (Identifiable &&) = default; + Identifiable() noexcept = default; + Identifiable(Identifiable &&) noexcept = default; + Identifiable & operator = (Identifiable &&) noexcept = default; Identifiable(const Identifiable &) = default; Identifiable & operator = (const Identifiable &) = default; - virtual ~Identifiable() { } + virtual ~Identifiable() noexcept = default; /** * Will produce the full demangled className diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 57e88873e81..a10cad70579 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -27,7 +27,7 @@ GrowableByteBuffer::allocate(uint32_t len) } void -GrowableByteBuffer::putBytes(const char* buffer, uint32_t length) +GrowableByteBuffer::putBytes(const void * buffer, uint32_t length) { char* buf = allocate(length); memcpy(buf, buffer, length); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index d32afeeabee..afbde04fb9b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -45,7 +45,7 @@ public: /** Adds the given buffer to this buffer. */ - void putBytes(const char* buffer, uint32_t length); + void putBytes(const void * buffer, uint32_t length); /** Adds a short to the buffer. diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index ff1d7e1d6ec..a758ca1fbbe 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -196,6 +196,8 @@ ProcessMemoryStats::create(uint64_t sizeEpsilon) i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str()); } std::sort(samples.begin(), samples.end()); + LOG(warning, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'", + sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str()); return samples[samples.size()/2]; } diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec index 98df4f71406..571b8b0ff4c 100644 --- a/standalone-container/vespa-standalone-container.spec +++ b/standalone-container/vespa-standalone-container.spec @@ -72,7 +72,6 @@ cp vespajlib/target/vespajlib.jar "$jars_dir" # Copy from submodules, so must be done separately cp zookeeper-server/zookeeper-server-common/target/zookeeper-server-common-jar-with-dependencies.jar "$jars_dir" -cp zookeeper-server/zookeeper-server-3.4/target/zookeeper-server-3.4-jar-with-dependencies.jar "$jars_dir" cp zookeeper-server/zookeeper-server-3.5/target/zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir" # Symlink to default version ln -s zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir"/zookeeper-server-jar-with-dependencies.jar diff --git a/storage/src/tests/common/message_sender_stub.cpp b/storage/src/tests/common/message_sender_stub.cpp index c127f9071e5..a82d45b0b99 100644 --- a/storage/src/tests/common/message_sender_stub.cpp +++ b/storage/src/tests/common/message_sender_stub.cpp @@ -22,9 +22,7 @@ MessageSenderStub::getLastCommand(bool verbose) const } std::string -MessageSenderStub::dumpMessage(const api::StorageMessage& msg, - bool includeAddress, - bool verbose) const +MessageSenderStub::dumpMessage(const api::StorageMessage& msg, bool includeAddress, bool verbose) const { std::ostringstream ost; @@ -67,9 +65,7 @@ MessageSenderStub::getLastReply(bool verbose) const throw std::logic_error("Expected reply where there was none"); } - return dumpMessage(*replies.back(), - true, - verbose); + return dumpMessage(*replies.back(),true, verbose); } diff --git a/storage/src/tests/distributor/distributor_message_sender_stub.h b/storage/src/tests/distributor/distributor_message_sender_stub.h index 7ebd4dee1ae..440dee70d48 100644 --- a/storage/src/tests/distributor/distributor_message_sender_stub.h +++ b/storage/src/tests/distributor/distributor_message_sender_stub.h @@ -5,6 +5,7 @@ #include <vespa/storage/distributor/distributormessagesender.h> #include <tests/common/message_sender_stub.h> #include <cassert> +#include <string> namespace storage { diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 81d91e2f08a..7c259f00899 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -19,7 +19,7 @@ vespalib::string fetch(int port, const vespalib::string &path) { auto crypto = vespalib::CryptoEngine::get_default(); auto socket = vespalib::SocketSpec::from_port(port).client_address().connect(); assert(socket.valid()); - auto conn = vespalib::SyncCryptoSocket::create(*crypto, std::move(socket), false); + auto conn = vespalib::SyncCryptoSocket::create_client(*crypto, std::move(socket), vespalib::SocketSpec::from_host_port("localhost", port)); vespalib::string http_req = vespalib::make_string("GET %s HTTP/1.1\r\n" "Host: localhost:%d\r\n" "\r\n", path.c_str(), port); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 64306fa7c24..4576f8a08f8 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1501,7 +1501,7 @@ TEST_F(FileStorManagerTest, visiting) { for (uint32_t i=3; i<docCount; ++i) { auto reply = std::dynamic_pointer_cast<api::BucketInfoReply>(top.getReply(i)); ASSERT_TRUE(reply.get()); - ASSERT_TRUE(reply->getResult().success()) << reply->getResult().toString(); + ASSERT_TRUE(reply->getResult().success()) << reply->getResult(); info = reply->getBucketInfo(); } diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp index 2bf7f7c3855..8c0f8853d2d 100644 --- a/storage/src/tests/persistence/processalltest.cpp +++ b/storage/src/tests/persistence/processalltest.cpp @@ -110,12 +110,11 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_returns_document_metadata_matc vespalib::string expected = "Persistence bucket BucketId(0x4000000000000004), partition 0\n" - " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" - " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" - " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" - " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" - " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n"; - + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 118\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 95\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 200\n"; EXPECT_EQ(expected, reply.getResults()); } @@ -145,16 +144,16 @@ TEST_F(ProcessAllHandlerTest, stat_bucket_request_can_returned_removed_entries) vespalib::string expected = "Persistence bucket BucketId(0x4000000000000004), partition 0\n" - " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" - " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 95\n" - " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" - " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 200\n" - " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" - " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 177\n" - " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" - " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 154\n" - " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n" - " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 130\n" + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" + " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 89\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" + " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 194\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 118\n" + " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 171\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 95\n" + " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 148\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 200\n" + " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 124\n" " Timestamp: 200, id:mail:testdoctype1:n=4:3619.html, gid(0x0400000092bb8d298934253a) (remove)\n" " Timestamp: 201, id:mail:testdoctype1:n=4:33113.html, gid(0x04000000b121a632741db368) (remove)\n" " Timestamp: 202, id:mail:testdoctype1:n=4:62608.html, gid(0x04000000ce878d2488413bc4) (remove)\n" @@ -191,16 +190,16 @@ TEST_F(ProcessAllHandlerTest, bucket_stat_request_can_return_all_put_entries_in_ vespalib::string expected = "Persistence bucket BucketId(0x4000000000000004), partition 0\n" - " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" - " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 95\n" - " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" - " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 200\n" - " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" - " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 177\n" - " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" - " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 154\n" - " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n" - " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 130\n"; + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 163\n" + " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 89\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 141\n" + " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 194\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 118\n" + " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 171\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 95\n" + " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 148\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 200\n" + " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 124\n"; EXPECT_EQ(expected, reply.getResults()); } diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index b7eb7fee3ec..20934d04eaa 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -20,6 +20,7 @@ #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> #include <gmock/gmock.h> #include <optional> #include <thread> @@ -337,7 +338,7 @@ int getTotalSerializedSize(const std::vector<document::Document::SP>& docs) { int total = 0; for (size_t i = 0; i < docs.size(); ++i) { - total += int(docs[i]->serialize()->getLength()); + total += int(docs[i]->serialize().size()); } return total; } diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index e92e2d4c3bf..1a4dc61a3ce 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -14,7 +14,7 @@ ReadBucketList::ReadBucketList(BucketSpace bucketSpace, spi::PartitionId partiti _partition(partition) { } -ReadBucketList::~ReadBucketList() { } +ReadBucketList::~ReadBucketList() = default; document::Bucket ReadBucketList::getBucket() const @@ -38,7 +38,7 @@ ReadBucketListReply::ReadBucketListReply(const ReadBucketList& cmd) _partition(cmd.getPartition()) { } -ReadBucketListReply::~ReadBucketListReply() { } +ReadBucketListReply::~ReadBucketListReply() = default; document::Bucket ReadBucketListReply::getBucket() const @@ -66,7 +66,7 @@ ReadBucketInfo::ReadBucketInfo(const document::Bucket &bucket) _bucket(bucket) { } -ReadBucketInfo::~ReadBucketInfo() { } +ReadBucketInfo::~ReadBucketInfo() = default; void ReadBucketInfo::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -92,7 +92,7 @@ ReadBucketInfoReply::ReadBucketInfoReply(const ReadBucketInfo& cmd) _bucket(cmd.getBucket()) { } -ReadBucketInfoReply::~ReadBucketInfoReply() { } +ReadBucketInfoReply::~ReadBucketInfoReply() = default; void ReadBucketInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "ReadBucketInfoReply()"; @@ -117,7 +117,7 @@ RepairBucketCommand::RepairBucketCommand(const document::Bucket &bucket, uint16_ setPriority(LOW); } -RepairBucketCommand::~RepairBucketCommand() { } +RepairBucketCommand::~RepairBucketCommand() = default; void RepairBucketCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -153,7 +153,7 @@ RepairBucketReply::RepairBucketReply(const RepairBucketCommand& cmd, const api:: _altered(false) { } -RepairBucketReply::~RepairBucketReply() { } +RepairBucketReply::~RepairBucketReply() = default; void RepairBucketReply::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -180,7 +180,7 @@ BucketDiskMoveCommand::BucketDiskMoveCommand(const document::Bucket &bucket, setPriority(LOW); } -BucketDiskMoveCommand::~BucketDiskMoveCommand() { } +BucketDiskMoveCommand::~BucketDiskMoveCommand() = default; void BucketDiskMoveCommand::setBucketId(const document::BucketId& id) @@ -208,14 +208,13 @@ BucketDiskMoveReply::BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, _dstDisk(cmd.getDstDisk()) { } -BucketDiskMoveReply::~BucketDiskMoveReply() { } +BucketDiskMoveReply::~BucketDiskMoveReply() = default; void BucketDiskMoveReply::print(std::ostream& out, bool, const std::string&) const { out << "BucketDiskMoveReply(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ", " << _bucketInfo << ", " - << getResult() << ")"; + << ", target " << _dstDisk << ", " << _bucketInfo << ", " << getResult() << ")"; } std::unique_ptr<api::StorageReply> @@ -236,7 +235,7 @@ InternalBucketJoinCommand::InternalBucketJoinCommand(const document::Bucket &buc // them higher than getting more bucket info lists. } -InternalBucketJoinCommand::~InternalBucketJoinCommand() { } +InternalBucketJoinCommand::~InternalBucketJoinCommand() = default; void InternalBucketJoinCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -255,7 +254,7 @@ InternalBucketJoinReply::InternalBucketJoinReply(const InternalBucketJoinCommand _bucketInfo(info) { } -InternalBucketJoinReply::~InternalBucketJoinReply() { } +InternalBucketJoinReply::~InternalBucketJoinReply() = default; void InternalBucketJoinReply::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index f73eb3ea36d..431c90b27f2 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -4,6 +4,7 @@ #include "bucketmessages.h" #include <vespa/vespalib/util/backtrace.h> #include <sstream> +#include <cassert> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".application.link"); @@ -141,7 +142,7 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) sendUp(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_down->onDown(msg)) { @@ -181,7 +182,7 @@ void StorageLink::sendUp(const shared_ptr<StorageMessage> & msg) sendDown(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_up->onUp(msg)) { diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index e143f4d8570..15a57c1e7ee 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -2,6 +2,7 @@ #include "simplemaintenancescanner.h" #include <vespa/storage/distributor/distributor_bucket_space.h> #include <ostream> +#include <cassert> namespace storage::distributor { @@ -16,10 +17,10 @@ SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucke { } -SimpleMaintenanceScanner::~SimpleMaintenanceScanner() {} +SimpleMaintenanceScanner::~SimpleMaintenanceScanner() = default; -SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() {} -SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() {} +SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(const PendingMaintenanceStats &) = default; SimpleMaintenanceScanner::PendingMaintenanceStats & SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (const PendingMaintenanceStats &) = default; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index 8e6494a588d..60c1137bd6d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.statbucket"); -namespace storage { -namespace distributor { +namespace storage::distributor { StatBucketOperation::StatBucketOperation( [[maybe_unused]] DistributorComponent& manager, @@ -21,7 +20,7 @@ StatBucketOperation::StatBucketOperation( { } -StatBucketOperation::~StatBucketOperation() {} +StatBucketOperation::~StatBucketOperation() = default; void StatBucketOperation::onClose(DistributorMessageSender& sender) @@ -36,8 +35,7 @@ StatBucketOperation::onStart(DistributorMessageSender& sender) { std::vector<uint16_t> nodes; - BucketDatabase::Entry entry( - _bucketSpace.getBucketDatabase().get(_command->getBucketId())); + BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(_command->getBucketId())); if (entry.valid()) { nodes = entry->getNodes(); @@ -103,5 +101,4 @@ StatBucketOperation::onReceive(DistributorMessageSender& sender, const std::shar } } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp index abd9778d72c..9e3230a0f34 100644 --- a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp +++ b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "throttlingoperationstarter.h" +#include <cassert> namespace storage::distributor { @@ -10,8 +11,7 @@ ThrottlingOperationStarter::ThrottlingOperation::~ThrottlingOperation() } bool -ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, - Priority priority) const +ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, Priority priority) const { uint32_t variablePending(_maxPending - _minPending); uint32_t maxPendingForPri(_minPending + variablePending*((255.0 - priority) / 255.0)); diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.cpp b/storage/src/vespa/storage/persistence/bucketprocessor.cpp index d4a570ee062..d6c549ef6f4 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.cpp +++ b/storage/src/vespa/storage/persistence/bucketprocessor.cpp @@ -4,6 +4,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/asciistream.h> #include <cassert> +#include <stdexcept> namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 350cdad791c..543accc80ae 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -6,22 +6,18 @@ namespace storage { FileStorHandler::FileStorHandler(MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(1, 1, sender, metrics, partitions, compReg)) + : _impl(std::make_unique<FileStorHandlerImpl>(1, 1, sender, metrics, partitions, compReg)) { } FileStorHandler::FileStorHandler(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(numThreads, numStripes, sender, metrics, partitions, compReg)) + : _impl(std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, sender, metrics, partitions, compReg)) { } -FileStorHandler::~FileStorHandler() -{ - delete _impl; -} - +FileStorHandler::~FileStorHandler() = default; void FileStorHandler::flush(bool flushMerges) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index ab3d03a5e9a..db294d7c39f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -35,7 +35,6 @@ namespace framework { class FileStorHandlerImpl; struct FileStorMetrics; struct MessageSender; -class MountPointList; struct ServiceLayerComponentRegister; class AbortBucketOperationsCommand; @@ -259,7 +258,7 @@ public: std::string dumpQueue(uint16_t disk) const; private: - FileStorHandlerImpl* _impl; + std::unique_ptr<FileStorHandlerImpl> _impl; }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index f773ee774bb..080446c1c92 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -879,15 +879,15 @@ FileStorHandlerImpl::MessageEntry::MessageEntry(MessageEntry && entry) noexcept _priority(entry._priority) { } -FileStorHandlerImpl::MessageEntry::~MessageEntry() { } +FileStorHandlerImpl::MessageEntry::~MessageEntry() = default; -FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads) +FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes) : metrics(0), _nextStripeId(0), - _stripes(numThreads, Stripe(owner, messageSender)), + _stripes(numStripes, Stripe(owner, messageSender)), state(FileStorHandler::AVAILABLE) { - assert(numThreads > 0); + assert(numStripes > 0); } FileStorHandlerImpl::Disk::Disk(Disk && rhs) noexcept diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 5fc592e11cb..44756ed5891 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -169,7 +169,7 @@ public: state.store(s, std::memory_order_relaxed); } - Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads); + Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes); Disk(Disk &&) noexcept; ~Disk(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index baa6523cbb2..b9fb85e0e80 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -38,11 +38,11 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi _bucketIdFactory(_component.getBucketIdFactory()), _configUri(configUri), _disks(), - _bucketOwnershipNotifier(new BucketOwnershipNotifier(_component, *this)), + _bucketOwnershipNotifier(std::make_unique<BucketOwnershipNotifier>(_component, *this)), _configFetcher(_configUri.getContext()), _threadLockCheckInterval(60), _failDiskOnError(false), - _metrics(new FileStorMetrics(_component.getLoadTypes()->getMetricLoadTypes())), + _metrics(std::make_unique<FileStorMetrics>(_component.getLoadTypes()->getMetricLoadTypes())), _threadMonitor(), _closed(false) { @@ -105,10 +105,10 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC _config = std::move(config); _disks.resize(_component.getDiskCount()); size_t numThreads = _config->numThreads; - size_t numStripes = std::min(2ul, numThreads); + size_t numStripes = std::min(4ul, numThreads); _metrics->initDiskMetrics(_disks.size(), _component.getLoadTypes()->getMetricLoadTypes(), numStripes, numThreads); - _filestorHandler.reset(new FileStorHandler(numThreads, numStripes, *this, *_metrics, _partitions, _compReg)); + _filestorHandler = std::make_unique<FileStorHandler>(numThreads, numStripes, *this, *_metrics, _partitions, _compReg); for (uint32_t i=0; i<_component.getDiskCount(); ++i) { if (_partitions[i].isUp()) { LOG(spam, "Setting up disk %u", i); @@ -849,12 +849,6 @@ FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPat _filestorHandler->getStatus(out, path); } -bool -FileStorManager::isMerging(const document::Bucket& bucket) const -{ - return _filestorHandler->isMerging(bucket); -} - namespace { struct Deactivator { StorBucketDatabase::Decision operator()(document::BucketId::Type, StorBucketDatabase::Entry& data) diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index bfc89a70a85..433b9ddbd39 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -36,11 +36,8 @@ namespace api { class StorageReply; } -class BucketMergeTest; -class DiskInfo; struct FileStorManagerTest; class ReadBucketList; -class ModifiedBucketCheckerThread; class BucketOwnershipNotifier; class AbortBucketOperationsCommand; @@ -48,65 +45,43 @@ class FileStorManager : public StorageLinkQueued, public framework::HtmlStatusReporter, public StateListener, private config::IFetcherCallback<vespa::config::content::StorFilestorConfig>, - private MessageSender + public MessageSender { - ServiceLayerComponentRegister& _compReg; - ServiceLayerComponent _component; - const spi::PartitionStateList& _partitions; - spi::PersistenceProvider& _providerCore; - ProviderErrorWrapper _providerErrorWrapper; - spi::PersistenceProvider* _provider; + ServiceLayerComponentRegister & _compReg; + ServiceLayerComponent _component; + const spi::PartitionStateList & _partitions; + spi::PersistenceProvider & _providerCore; + ProviderErrorWrapper _providerErrorWrapper; + spi::PersistenceProvider * _provider; const document::BucketIdFactory& _bucketIdFactory; - config::ConfigUri _configUri; + config::ConfigUri _configUri; typedef std::vector<DiskThread::SP> DiskThreads; - std::vector<DiskThreads> _disks; + std::vector<DiskThreads> _disks; std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; std::unique_ptr<vespa::config::content::StorFilestorConfig> _config; config::ConfigFetcher _configFetcher; - uint32_t _threadLockCheckInterval; // In seconds - bool _failDiskOnError; - int _killSignal; + uint32_t _threadLockCheckInterval; // In seconds + bool _failDiskOnError; std::shared_ptr<FileStorMetrics> _metrics; std::unique_ptr<FileStorHandler> _filestorHandler; - lib::ClusterState _lastState; - struct ReplyHolder { - int refCount; - std::unique_ptr<api::StorageReply> reply; - - ReplyHolder(int rc, std::unique_ptr<api::StorageReply> r) - : refCount(rc), reply(std::move(r)) {}; - }; - - std::map<api::StorageMessage::Id, - std::shared_ptr<ReplyHolder> > _splitMessages; - vespalib::Lock _splitLock; mutable vespalib::Monitor _threadMonitor; // Notify to stop sleeping - bool _closed; - - FileStorManager(const FileStorManager &); - FileStorManager& operator=(const FileStorManager &); + bool _closed; - std::vector<DiskThreads> getThreads() { return _disks; } - - friend class BucketMergeTest; friend struct FileStorManagerTest; - friend class MessageTest; public: - explicit FileStorManager(const config::ConfigUri &, - const spi::PartitionStateList&, - spi::PersistenceProvider&, - ServiceLayerComponentRegister&); - ~FileStorManager(); + FileStorManager(const config::ConfigUri &, const spi::PartitionStateList&, + spi::PersistenceProvider&, ServiceLayerComponentRegister&); + FileStorManager(const FileStorManager &) = delete; + FileStorManager& operator=(const FileStorManager &) = delete; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + ~FileStorManager() override; - // Return true if we are currently merging the given bucket. - bool isMerging(const document::Bucket& bucket) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; FileStorHandler& getFileStorHandler() { return *_filestorHandler; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 37e1d818bb8..e5e358cbb60 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -506,18 +506,11 @@ MergeHandler::fetchLocalData( assert(doc != 0); assertContainedInBucket(doc->getId(), bucket, idFactory); e._docName = doc->getId().toString(); - { - vespalib::nbostream stream; - doc->serializeHeader(stream); - e._headerBlob.resize(stream.size()); - memcpy(&e._headerBlob[0], stream.peek(), stream.size()); - } - { - vespalib::nbostream stream; - doc->serializeBody(stream); - e._bodyBlob.resize(stream.size()); - memcpy(&e._bodyBlob[0], stream.peek(), stream.size()); - } + vespalib::nbostream stream; + doc->serialize(stream); + e._headerBlob.resize(stream.size()); + memcpy(&e._headerBlob[0], stream.peek(), stream.size()); + e._bodyBlob.clear(); } else { const DocumentId* docId = docEntry.getDocumentId(); assert(docId != 0); @@ -556,11 +549,11 @@ MergeHandler::deserializeDiffDocument( const api::ApplyBucketDiffCommand::Entry& e, const document::DocumentTypeRepo& repo) const { - Document::UP doc(new Document); - using document::ByteBuffer; - ByteBuffer hbuf(&e._headerBlob[0], e._headerBlob.size()); + auto doc = std::make_unique<Document>(); + vespalib::nbostream hbuf(&e._headerBlob[0], e._headerBlob.size()); if (e._bodyBlob.size() > 0) { - ByteBuffer bbuf(&e._bodyBlob[0], e._bodyBlob.size()); + // TODO Remove this branch and add warning on error. + vespalib::nbostream bbuf(&e._bodyBlob[0], e._bodyBlob.size()); doc->deserialize(repo, hbuf, bbuf); } else { doc->deserialize(repo, hbuf); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 7d0ce26b83d..dd44c96555b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -795,8 +795,8 @@ PersistenceThread::handleCommand(api::StorageCommand& msg) { _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker.get() != 0) { - if (mtracker->getReply().get() != 0) { + if (mtracker) { + if (mtracker->getReply()) { mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); } else { msg.getTrace().getRoot().addChild(_context.getTrace().getRoot()); diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 73873e78032..1aafe58af6c 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -7,11 +7,12 @@ vespa_add_library(storage_storageserver changedbucketownershiphandler.cpp communicationmanager.cpp communicationmanagermetrics.cpp - configurable_bucket_resolver.cpp config_logging.cpp + configurable_bucket_resolver.cpp distributornode.cpp distributornodecontext.cpp documentapiconverter.cpp + fnet_metrics_wrapper.cpp fnetlistener.cpp mergethrottler.cpp messagesink.cpp diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp new file mode 100644 index 00000000000..476ececd078 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fnet_metrics_wrapper.h" + +namespace storage { + +FnetMetricsWrapper::FnetMetricsWrapper(metrics::MetricSet* owner) + : metrics::MetricSet("fnet", {}, "transport layer metrics", owner), + _num_connections("num-connections", {}, "total number of connection objects", this) +{ +} + +FnetMetricsWrapper::~FnetMetricsWrapper() = default; + +void +FnetMetricsWrapper::update_metrics() +{ + _num_connections.set(FNET_Connection::get_num_connections()); +} + +} diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h new file mode 100644 index 00000000000..dacb9e9f52c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h @@ -0,0 +1,22 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/metrics/metrics.h> +#include <vespa/fnet/connection.h> + +namespace storage { + +// Simple wrapper around low-level fnet network metrics +class FnetMetricsWrapper : public metrics::MetricSet +{ +private: + metrics::LongValueMetric _num_connections; + +public: + explicit FnetMetricsWrapper(metrics::MetricSet* owner); + ~FnetMetricsWrapper() override; + void update_metrics(); +}; + +} diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index 6fc9795993e..b6bceabf7a1 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -77,7 +77,7 @@ OpsLogger::onPutReply(const std::shared_ptr<api::PutReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tPUT\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -94,7 +94,7 @@ OpsLogger::onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tUPDATE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -111,7 +111,7 @@ OpsLogger::onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tREMOVE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -128,7 +128,7 @@ OpsLogger::onGetReply(const std::shared_ptr<api::GetReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tGET\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index f0e64f0dfd1..6ed5c1ab650 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -16,37 +16,6 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; -DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("document_serialization", {{"docserialization"}}, - "Counts of document serialization of various types", owner), - usedCachedSerializationCount( - "cached_serialization_count", {{"docserialization"}}, - "Number of times we didn't need to serialize the document as " - "we already had serialized version cached", this), - compressedDocumentCount( - "compressed_serialization_count", {{"docserialization"}}, - "Number of times we compressed document when serializing", - this), - compressionDidntHelpCount( - "compressed_didnthelp_count", {{"docserialization"}}, - "Number of times we compressed document when serializing, but " - "the compressed version was bigger, so it was dumped", this), - uncompressableCount( - "uncompressable_serialization_count", {{"docserialization"}}, - "Number of times we didn't attempt compression as document " - "had already been tagged uncompressable", this), - serializedUncompressed( - "uncompressed_serialization_count", {{"docserialization"}}, - "Number of times we serialized a document uncompressed", this), - inputWronglySerialized( - "input_wrongly_serialized_count", {{"docserialization"}}, - "Number of times we reserialized a document because the " - "compression it had in cache did not match what was configured", - this) -{} - -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; - StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, "Metrics for VDS applications"), @@ -54,34 +23,20 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this), - tls_metrics(this) + tls_metrics(this), + fnet_metrics(this) {} StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { - document::SerializableArray::Statistics stats( - document::SerializableArray::getStatistics()); - - documentSerialization.usedCachedSerializationCount.set( - stats._usedCachedSerializationCount); - documentSerialization.compressedDocumentCount.set( - stats._compressedDocumentCount); - documentSerialization.compressionDidntHelpCount.set( - stats._compressionDidntHelpCount); - documentSerialization.uncompressableCount.set( - stats._uncompressableCount); - documentSerialization.serializedUncompressed.set( - stats._serializedUncompressed); - documentSerialization.inputWronglySerialized.set( - stats._inputWronglySerialized); // Delta snapshotting is destructive, so if an explicit snapshot is triggered // (instead of just regular periodic snapshots), some events will effectively // be erased from history. This will no longer be a problem once we move to a // metrics system built around absolute (rather than derived) values. tls_metrics.update_metrics_with_snapshot_delta(); + fnet_metrics.update_metrics(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index e9378010540..a315e974c01 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -3,7 +3,7 @@ #pragma once #include "tls_statistics_metrics_wrapper.h" - +#include "fnet_metrics_wrapper.h" #include <vespa/metrics/metrics.h> namespace storage { @@ -17,21 +17,8 @@ public: metrics::LongValueMetric highpri; metrics::LongValueMetric veryhighpri; - MessageMemoryUseMetricSet(metrics::MetricSet* owner); - ~MessageMemoryUseMetricSet(); -}; - -struct DocumentSerializationMetricSet : public metrics::MetricSet -{ - metrics::LongCountMetric usedCachedSerializationCount; - metrics::LongCountMetric compressedDocumentCount; - metrics::LongCountMetric compressionDidntHelpCount; - metrics::LongCountMetric uncompressableCount; - metrics::LongCountMetric serializedUncompressed; - metrics::LongCountMetric inputWronglySerialized; - - DocumentSerializationMetricSet(metrics::MetricSet* owner); - ~DocumentSerializationMetricSet(); + explicit MessageMemoryUseMetricSet(metrics::MetricSet* owner); + ~MessageMemoryUseMetricSet() override; }; struct StorageMetricSet : public metrics::MetricSet @@ -39,12 +26,12 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse; MessageMemoryUseMetricSet memoryUse_messages; metrics::LongValueMetric memoryUse_visiting; - DocumentSerializationMetricSet documentSerialization; TlsStatisticsMetricsWrapper tls_metrics; + FnetMetricsWrapper fnet_metrics; StorageMetricSet(); - ~StorageMetricSet(); + ~StorageMetricSet() override; void updateMetrics(); }; diff --git a/storage/src/vespa/storage/tools/analyzedistribution.cpp b/storage/src/vespa/storage/tools/analyzedistribution.cpp index 52c26c4bb17..472aa63fff6 100644 --- a/storage/src/vespa/storage/tools/analyzedistribution.cpp +++ b/storage/src/vespa/storage/tools/analyzedistribution.cpp @@ -132,7 +132,7 @@ Node::Node(const lib::NodeState& dstate, const lib::NodeState& sstate, uint32_t disks.push_back(Disk(storageState.getDiskState(i))); } } -Node::~Node() {} +Node::~Node() = default; struct Distribution { std::vector<Node> nodes; diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 7c69a232af0..80e74e890a1 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -2,7 +2,7 @@ #include "recoveryvisitor.h" - +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -36,10 +36,9 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, { vespalib::LockGuard guard(_mutex); - LOG(debug, "Visitor %s handling block of %zu documents.", - _id.c_str(), entries.size()); + LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), entries.size()); - documentapi::DocumentListMessage* cmd = NULL; + documentapi::DocumentListMessage* cmd = nullptr; { CommandMap::iterator iter = _activeCommands.find(bid); @@ -71,7 +70,7 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, } } - hitCounter.addHit(doc->getId(), doc->serialize()->getLength()); + hitCounter.addHit(doc->getId(), doc->serialize().size()); int64_t timestamp = doc->getLastModified(); cmd->getDocuments().push_back(documentapi::DocumentListMessage::Entry( diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index e68a8fdbc8c..1da2acfed9c 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -12,16 +12,13 @@ #include "visitor.h" #include <vespa/storageapi/message/datagram.h> -namespace documentapi { -class DocumentListMessage; -} +namespace documentapi { class DocumentListMessage; } namespace storage { class RecoveryVisitor : public Visitor { public: - RecoveryVisitor(StorageComponent&, - const vdslib::Parameters& params); + RecoveryVisitor(StorageComponent&, const vdslib::Parameters& params); private: void handleDocuments(const document::BucketId& bucketId, @@ -43,12 +40,11 @@ struct RecoveryVisitorFactory : public VisitorFactory { VisitorEnvironment::UP makeVisitorEnvironment(StorageComponent&) override { - return VisitorEnvironment::UP(new VisitorEnvironment); + return std::make_unique<VisitorEnvironment>(); }; Visitor* - makeVisitor(StorageComponent& c, VisitorEnvironment&, - const vdslib::Parameters& params) override + makeVisitor(StorageComponent& c, VisitorEnvironment&, const vdslib::Parameters& params) override { return new RecoveryVisitor(c, params); } diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 4b213dff1d5..bdd066e8a4a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -44,17 +44,12 @@ Visitor::HitCounter::addHit(const document::DocumentId& , uint32_t size) } void -Visitor::HitCounter::updateVisitorStatistics( - vdslib::VisitorStatistics& statistics) +Visitor::HitCounter::updateVisitorStatistics(vdslib::VisitorStatistics& statistics) { - statistics.setDocumentsReturned( - statistics.getDocumentsReturned() + _firstPassHits); - statistics.setBytesReturned( - statistics.getBytesReturned() + _firstPassBytes); - statistics.setSecondPassDocumentsReturned( - statistics.getSecondPassDocumentsReturned() + _secondPassHits); - statistics.setSecondPassBytesReturned( - statistics.getSecondPassBytesReturned() + _secondPassBytes); + statistics.setDocumentsReturned(statistics.getDocumentsReturned() + _firstPassHits); + statistics.setBytesReturned(statistics.getBytesReturned() + _firstPassBytes); + statistics.setSecondPassDocumentsReturned(statistics.getSecondPassDocumentsReturned() + _secondPassHits); + statistics.setSecondPassBytesReturned(statistics.getSecondPassBytesReturned() + _secondPassBytes); } Visitor::VisitorTarget::MessageMeta::MessageMeta( @@ -68,8 +63,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::MessageMeta( - Visitor::VisitorTarget::MessageMeta&& rhs) noexcept +Visitor::VisitorTarget::MessageMeta::MessageMeta(Visitor::VisitorTarget::MessageMeta&& rhs) noexcept : messageId(rhs.messageId), retryCount(rhs.retryCount), memoryUsage(rhs.memoryUsage), @@ -78,9 +72,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::~MessageMeta() -{ -} +Visitor::VisitorTarget::MessageMeta::~MessageMeta() = default; Visitor::VisitorTarget::MessageMeta& Visitor::VisitorTarget::MessageMeta::operator=( diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index b90153c9517..0cfd2160497 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -24,8 +24,7 @@ ProtocolSerialization4_2::ProtocolSerialization4_2( { } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -41,14 +40,12 @@ ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); bool headerOnly(SH::getBoolean(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RemoveCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -62,13 +59,12 @@ ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const document::DocumentId did(SH::getString(buf)); document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); - api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp)); + auto msg = std::make_unique<api::RemoveCommand>(bucket, did, timestamp); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RevertCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putInt(msg.getRevertTokens().size()); @@ -86,13 +82,12 @@ ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const for (uint32_t i=0, n=tokens.size(); i<n; ++i) { tokens[i] = SH::getLong(buf); } - api::RevertCommand::UP msg(new api::RevertCommand(bucket, tokens)); + auto msg = std::make_unique<api::RevertCommand>(bucket, tokens); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); @@ -102,13 +97,12 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::MergeBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -135,14 +129,12 @@ ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp(SH::getLong(buf)); - api::MergeBucketCommand::UP msg( - new api::MergeBucketCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::MergeBucketCommand>(bucket, nodes, timestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -174,8 +166,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp = SH::getLong(buf); - api::GetBucketDiffCommand::UP msg( - new api::GetBucketDiffCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::GetBucketDiffCommand>(bucket, nodes, timestamp); std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -190,8 +181,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -201,18 +191,15 @@ void ProtocolSerialization4_2::onEncode( buf.putBoolean(nodes[i].sourceOnly); } buf.putInt(msg.getMaxBufferSize()); - const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( - msg.getDiff()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { onEncodeDiffEntry(buf, entries[i]._entry); buf.putString(entries[i]._docName); buf.putInt(entries[i]._headerBlob.size()); - buf.putBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); buf.putInt(entries[i]._bodyBlob.size()); - buf.putBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onEncodeBucketInfoCommand(buf, msg); } @@ -231,8 +218,7 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } uint32_t maxBufferSize(SH::getInt(buf)); - api::ApplyBucketDiffCommand::UP msg( - new api::ApplyBucketDiffCommand(bucket, nodes, maxBufferSize)); + auto msg = std::make_unique<api::ApplyBucketDiffCommand>(bucket, nodes, maxBufferSize); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -248,15 +234,13 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const buf.incPos(headerSize); } entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); uint32_t bodySize = SH::getInt(buf); if (bodySize > buf.getRemaining()) { buf.incPos(bodySize); } entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onDecodeBucketInfoCommand(buf, *msg); return msg; @@ -274,11 +258,9 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { - api::RequestBucketInfoReply::UP msg(new api::RequestBucketInfoReply( - static_cast<const api::RequestBucketInfoCommand&>(cmd))); + auto msg = std::make_unique<api::RequestBucketInfoReply>(static_cast<const api::RequestBucketInfoCommand&>(cmd)); api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -294,8 +276,7 @@ ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { putBucket(msg.getBucket(), buf); putBucketInfo(msg.getBucketInfo(), buf); @@ -307,30 +288,25 @@ ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::BucketInfo info(getBucketInfo(buf)); - api::NotifyBucketChangeCommand::UP msg( - new api::NotifyBucketChangeCommand(bucket, info)); + auto msg = std::make_unique<api::NotifyBucketChangeCommand>(bucket, info); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeReply& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { onEncodeReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd,BBuf& buf) const { - api::NotifyBucketChangeReply::UP msg(new api::NotifyBucketChangeReply( - static_cast<const api::NotifyBucketChangeCommand&>(cmd))); + auto msg = std::make_unique<api::NotifyBucketChangeReply>(static_cast<const api::NotifyBucketChangeCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::SplitBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putByte(msg.getMinSplitBits()); @@ -344,7 +320,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(bucket)); + auto msg = std::make_unique<api::SplitBucketCommand>(bucket); msg->setMinSplitBits(SH::getByte(buf)); msg->setMaxSplitBits(SH::getByte(buf)); msg->setMinByteSize(SH::getInt(buf)); @@ -353,30 +329,24 @@ ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateCommand&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateCommand&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const { - throw vespalib::IllegalStateException("Unsupported deserialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateReply&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateReply&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, - BBuf&) const +ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, BBuf&) const { throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } @@ -404,11 +374,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateVisitorCommand& buf.putBoolean(msg.getFieldSet() == "[header]"); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); } @@ -420,8 +386,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -450,7 +415,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); msg->setVisitorDispatcherVersion(42); @@ -458,8 +423,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const } void -ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::DestroyVisitorCommand& msg) const +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { buf.putString(msg.getInstanceId()); onEncodeCommand(buf, msg); @@ -469,7 +433,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const { vespalib::stringref instanceId = SH::getString(buf); - api::DestroyVisitorCommand::UP msg(new api::DestroyVisitorCommand(instanceId)); + auto msg = std::make_unique<api::DestroyVisitorCommand>(instanceId); onDecodeCommand(buf, *msg); return msg; } @@ -483,7 +447,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { - api::DestroyVisitorReply::UP msg(new api::DestroyVisitorReply(static_cast<const api::DestroyVisitorCommand&>(cmd))); + auto msg = std::make_unique<api::DestroyVisitorReply>(static_cast<const api::DestroyVisitorCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } @@ -502,8 +466,7 @@ ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const vespalib::stringref documentSelection = SH::getString(buf); document::Bucket bucket = getBucket(buf); - api::RemoveLocationCommand::UP msg; - msg.reset(new api::RemoveLocationCommand(documentSelection, bucket)); + auto msg = std::make_unique<api::RemoveLocationCommand>(documentSelection, bucket); onDecodeCommand(buf, *msg); return msg; } @@ -517,7 +480,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - api::RemoveLocationReply::UP msg(new api::RemoveLocationReply(static_cast<const api::RemoveLocationCommand&>(cmd))); + auto msg = std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); return msg; } @@ -525,15 +488,13 @@ ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf // Utility functions for serialization void -ProtocolSerialization4_2::onEncodeBucketInfoCommand( - GBBuf& buf, const api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onEncodeBucketInfoCommand(GBBuf& buf, const api::BucketInfoCommand& msg) const { onEncodeCommand(buf, msg); } void -ProtocolSerialization4_2::onDecodeBucketInfoCommand( - BBuf& buf, api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onDecodeBucketInfoCommand(BBuf& buf, api::BucketInfoCommand& msg) const { onDecodeCommand(buf, msg); } @@ -547,8 +508,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const } void -ProtocolSerialization4_2::onEncodeDiffEntry( - GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onEncodeDiffEntry(GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const { buf.putLong(entry._timestamp); SH::putGlobalId(entry._gid, buf); @@ -559,8 +519,7 @@ ProtocolSerialization4_2::onEncodeDiffEntry( } void -ProtocolSerialization4_2::onDecodeDiffEntry( - BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onDecodeDiffEntry(BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const { entry._timestamp = SH::getLong(buf); entry._gid = SH::getGlobalId(buf); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index b0a1685ed8c..0b1f66127ba 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -62,32 +62,26 @@ ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::SetBucketStateCommand::BUCKET_STATE state( - static_cast<api::SetBucketStateCommand::BUCKET_STATE>( - SH::getByte(buf))); - api::SetBucketStateCommand::UP msg( - new api::SetBucketStateCommand(bucket, state)); + static_cast<api::SetBucketStateCommand::BUCKET_STATE>(SH::getByte(buf))); + auto msg = std::make_unique<api::SetBucketStateCommand>(bucket, state); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::SetBucketStateReply& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { - api::SetBucketStateReply::UP msg(new api::SetBucketStateReply( - static_cast<const api::SetBucketStateCommand&>(cmd))); + auto msg = std::make_unique<api::SetBucketStateReply>(static_cast<const api::SetBucketStateCommand&>(cmd)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -103,15 +97,13 @@ ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); std::string fieldSet(SH::getString(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, fieldSet, beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, fieldSet, beforeTimestamp); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void -ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateVisitorCommand& msg) const +ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { putBucketSpace(msg.getBucketSpace(), buf); buf.putString(msg.getLibraryName()); @@ -133,11 +125,7 @@ ProtocolSerialization5_1::onEncode( buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); @@ -152,8 +140,7 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -182,17 +169,16 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); SH::getInt(buf); // Unused msg->setMaxBucketsPerVisitor(SH::getInt(buf)); msg->setVisitorDispatcherVersion(50); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putBoolean(msg.getActive()); @@ -204,10 +190,10 @@ ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); bool setActive = SH::getBoolean(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); msg->setActive(setActive); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index ea002ab98ed..9751fd1be98 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -85,7 +85,7 @@ std::shared_ptr<document::Document> get_document(const protobuf::Document& src_d const document::DocumentTypeRepo& type_repo) { if (!src_doc.payload().empty()) { - document::ByteBuffer doc_buf(src_doc.payload().data(), src_doc.payload().size()); + vespalib::nbostream doc_buf(src_doc.payload().data(), src_doc.payload().size()); return std::make_shared<document::Document>(type_repo, doc_buf); } return std::shared_ptr<document::Document>(); diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h index 08cec601cce..24cc3e4371d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -87,9 +87,9 @@ public: if (size == 0) { return document::Document::UP(); } else { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + vespalib::nbostream stream(buf.getBufferAtPos(), size); buf.incPos(size); - return document::Document::UP(new document::Document(repo, bbuf)); + return std::make_unique<document::Document>(repo, stream); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp index 469c6a41bc7..596af6a79d4 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp @@ -3,6 +3,8 @@ #include "storagereply.h" #include "storagecommand.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/objects/nbostream.h> + using vespalib::alloc::Alloc; using vespalib::IllegalStateException; @@ -17,8 +19,8 @@ StorageReply::StorageReply(mbus::BlobRef data, const ProtocolSerialization& seri _reply() { memcpy(_buffer.get(), data.data(), _sz); - document::ByteBuffer buf(data.data(), _sz); - buf.getIntNetwork(reinterpret_cast<int32_t&>(_mbusType)); + vespalib::nbostream nbo(data.data(), _sz); + nbo >> _mbusType; } StorageReply::StorageReply(api::StorageReply::SP reply) diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index ee8aff31914..de47a52ca9d 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -3,6 +3,7 @@ #include "bucket.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/array.hpp> #include <ostream> #include <iterator> @@ -319,7 +320,7 @@ ApplyBucketDiffCommand::Entry::print( << "), headerBlob(" << _headerBlob.size() << "), bodyBlob(" << _bodyBlob.size() << ")"; if (_headerBlob.size() > 0) { - document::ByteBuffer buf(&_headerBlob[0], + vespalib::nbostream buf(&_headerBlob[0], _headerBlob.size()); if (_repo) { document::Document doc(*_repo, buf); diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index f6185d9f8e7..45af2296e8a 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -14,12 +14,12 @@ #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/base/globalid.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/util/array.h> #include <vespa/storageapi/defs.h> namespace document { class DocumentTypeRepo; } -namespace storage { -namespace api { +namespace storage::api { /** * @class CreateBucketCommand @@ -254,6 +254,8 @@ public: GetBucketDiffCommand::Entry _entry; vespalib::string _docName; std::vector<char> _headerBlob; + // TODO: In theory the body blob could be removed now as all is in one blob + // That will enable simplification of code in document. std::vector<char> _bodyBlob; const document::DocumentTypeRepo *_repo; @@ -282,7 +284,7 @@ public: ApplyBucketDiffCommand(const document::Bucket &bucket, const std::vector<Node>& nodes, uint32_t maxBufferSize); - ~ApplyBucketDiffCommand(); + ~ApplyBucketDiffCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } const std::vector<Entry>& getDiff() const { return _diff; } @@ -482,5 +484,4 @@ public: DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply) }; -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp index 1463f42abeb..7fd789a8c81 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -60,7 +60,7 @@ PutCommand::print(std::ostream& out, bool verbose, const std::string& indent) co { out << "Put(" << getBucketId() << ", " << _doc->getId() << ", timestamp " << _timestamp << ", size " - << _doc->serialize()->getLength() << ")"; + << _doc->serialize().size() << ")"; if (verbose) { out << " {\n" << indent << " "; _doc->print(out, verbose, indent + " "); diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index aeb58f30fb4..e531e73fea5 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -65,7 +65,7 @@ CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) { } -CreateVisitorCommand::~CreateVisitorCommand() {} +CreateVisitorCommand::~CreateVisitorCommand() = default; document::Bucket CreateVisitorCommand::getBucket() const @@ -141,8 +141,7 @@ DestroyVisitorCommand::DestroyVisitorCommand(vespalib::stringref instanceId) } void -DestroyVisitorCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorCommand(" << _instanceId << ")"; if (verbose) { @@ -157,8 +156,7 @@ DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd) } void -DestroyVisitorReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorReply()"; if (verbose) { @@ -175,12 +173,10 @@ VisitorInfoCommand::VisitorInfoCommand() { } -VisitorInfoCommand::~VisitorInfoCommand() { -} +VisitorInfoCommand::~VisitorInfoCommand() = default; void -VisitorInfoCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoCommand("; if (_completed) { out << "completed"; } @@ -205,8 +201,7 @@ VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd) } void -VisitorInfoReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoReply("; if (_completed) { out << "completed"; } diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp index 68fbca75393..d5c8cb7da68 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp @@ -3,8 +3,7 @@ #include "returncode.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { ReturnCode::ReturnCode() : _result(OK), @@ -13,59 +12,33 @@ ReturnCode::ReturnCode() ReturnCode::ReturnCode(const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (const ReturnCode &) = default; -ReturnCode & ReturnCode::operator = (ReturnCode &&) = default; -ReturnCode::~ReturnCode() {} +ReturnCode & ReturnCode::operator = (ReturnCode &&) noexcept = default; +ReturnCode::~ReturnCode() = default; ReturnCode::ReturnCode(Result result, vespalib::stringref msg) : _result(result), _message(msg) {} -ReturnCode::ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer) - : _result(OK), - _message() -{ - deserialize(repo, buffer); -} - -void ReturnCode:: -onDeserialize(const document::DocumentTypeRepo &, document::ByteBuffer& buffer) -{ - int32_t result; - buffer.getInt(result); - _result = static_cast<Result>(result); - int32_t size; - buffer.getInt(size); - const char * p = buffer.getBufferAtPos(); - buffer.incPos(size); - _message.assign(p, size); -} - -void ReturnCode::onSerialize(document::ByteBuffer& buffer) const -{ - buffer.putInt(_result); - buffer.putInt(_message.size()); - buffer.putBytes(_message.c_str(), _message.size()); -} - -size_t ReturnCode::getSerializedSize() const -{ - return 2 * sizeof(int32_t) + _message.size(); +vespalib::string ReturnCode::getResultString(Result result) { + return documentapi::DocumentProtocol::getErrorName(result); } -void -ReturnCode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "ReturnCode(" << ReturnCode::getResultString(getResult()); - if (getMessage().size() > 0) out << ", " << getMessage(); - out << ")"; +vespalib::string +ReturnCode::toString() const { + vespalib::string ret = "ReturnCode("; + ret += getResultString(_result); + if ( ! _message.empty()) { + ret += ", "; + ret += _message; + } + ret += ")"; + return ret; } -vespalib::string ReturnCode::getResultString(Result result) { - return documentapi::DocumentProtocol::getErrorName(result); +std::ostream & +operator << (std::ostream & os, const ReturnCode & returnCode) { + return os << returnCode.toString(); } bool @@ -173,5 +146,4 @@ ReturnCode::isBucketDisappearance() const } } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h index ccd95a81aa3..0149ae29a05 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h @@ -10,20 +10,12 @@ #pragma once -#include <vespa/vespalib/util/printable.h> -#include <vespa/document/util/serializable.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <iosfwd> -namespace document { - class ByteBuffer; -} -namespace storage { -namespace api { +namespace storage::api { -class ReturnCode : public document::Deserializable, - public vespalib::Printable { +class ReturnCode { public: typedef documentapi::DocumentProtocol Protocol; @@ -68,31 +60,20 @@ public: private: Result _result; vespalib::string _message; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; - void onSerialize(document::ByteBuffer& buffer) const override; - public: ReturnCode(); explicit ReturnCode(Result result, vespalib::stringref msg = ""); - ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer); ReturnCode(const ReturnCode &); ReturnCode & operator = (const ReturnCode &); - ReturnCode(ReturnCode &&) = default; - ReturnCode & operator = (ReturnCode &&); + ReturnCode(ReturnCode &&) noexcept = default; + ReturnCode & operator = (ReturnCode &&) noexcept; ~ReturnCode(); - ReturnCode* clone() const override { return new ReturnCode(*this); } - - size_t getSerializedSize() const override; - const vespalib::string& getMessage() const { return _message; } void setMessage(vespalib::stringref message) { _message = message; } Result getResult() const { return _result; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - /** * Translate from status code to human-readable string * @param result Status code returned from getResult() @@ -121,8 +102,9 @@ public: bool isShutdownRelated() const; bool isBucketDisappearance() const; bool isNonCriticalForIntegrityChecker() const; + vespalib::string toString() const; }; +std::ostream & operator << (std::ostream & os, const ReturnCode & returnCode); -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp index 86f7cb6e16d..81cdadb3623 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp @@ -4,8 +4,7 @@ #include "storagecommand.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()), @@ -19,16 +18,14 @@ StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) setTransportContext(cmd.getTransportContext()); } -StorageReply::~StorageReply() { } +StorageReply::~StorageReply() = default; void StorageReply::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; - out << "StorageReply(" << _type.getName() << ", " - << _result.toString() << ")"; + out << "StorageReply(" << _type.getName() << ", " << _result << ")"; } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h index 4219f3e28cd..1a3bbe35eb4 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h @@ -15,8 +15,7 @@ #include "returncode.h" #include "storagemessage.h" -namespace storage { -namespace api { +namespace storage::api { class StorageCommand; @@ -28,7 +27,7 @@ protected: ReturnCode code = ReturnCode(ReturnCode::OK)); public: - ~StorageReply(); + ~StorageReply() override; DECLARE_POINTER_TYPEDEFS(StorageReply); void setResult(const ReturnCode& r) { _result = r; } @@ -37,6 +36,4 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; -} // api -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp index 337709cc591..10691d37b9e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp @@ -77,7 +77,7 @@ HttpUrlPath::print(std::ostream& out, bool, const std::string&) const if (!_attributes.empty()) { out << "?"; size_t cnt = 0; - for (const auto attr: _attributes) { + for (const auto &attr: _attributes) { if (cnt++ > 0) { out << "&"; } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 94026ffaa19..eb1cc7f0256 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -1108,7 +1108,7 @@ SearchVisitor::generateGroupingResults() vespalib::NBOSerializer nos(os); grouping.serialize(nos); vespalib::MallocPtr blob(os.size()); - memcpy(blob, os.c_str(), os.size()); + memcpy(blob, os.data(), os.size()); searchResult.getGroupingList().add(grouping.getId(), blob); } } diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 22e1bb0e9ca..a3fac22a93d 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -253,6 +253,30 @@ </plugins> </build> </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> </profiles> <build> diff --git a/travis/travis-build-full.sh b/travis/travis-build-full.sh index 1715c9e2288..62821e0e06d 100755 --- a/travis/travis-build-full.sh +++ b/travis/travis-build-full.sh @@ -11,7 +11,7 @@ source /etc/profile.d/enable-rh-maven35.sh ccache --max-size=1250M ccache --set-config=compression=true -ccache --print-config +ccache -p cd ${SOURCE_DIR} env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh java diff --git a/vbench/src/vbench/core/socket.cpp b/vbench/src/vbench/core/socket.cpp index 822b96b2c07..0431b6889a8 100644 --- a/vbench/src/vbench/core/socket.cpp +++ b/vbench/src/vbench/core/socket.cpp @@ -29,7 +29,8 @@ Socket::Socket(SyncCryptoSocket::UP socket) } Socket::Socket(CryptoEngine &crypto, const string &host, int port) - : _socket(SyncCryptoSocket::create(crypto, connect(host, port), false)), + : _socket(SyncCryptoSocket::create_client(crypto, connect(host, port), + vespalib::SocketSpec::from_host_port(host, port))), _input(), _output(), _taint(), diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h index 0e8848e8292..4b5721c0fb7 100644 --- a/vbench/src/vbench/core/socket.h +++ b/vbench/src/vbench/core/socket.h @@ -52,7 +52,7 @@ struct ServerSocket { Stream::UP accept(CryptoEngine &crypto) { vespalib::SocketHandle handle = server_socket.accept(); if (handle.valid()) { - return std::make_unique<Socket>(SyncCryptoSocket::create(crypto, std::move(handle), true)); + return std::make_unique<Socket>(SyncCryptoSocket::create_server(crypto, std::move(handle))); } else { return Stream::UP(); } diff --git a/vdslib/src/tests/container/parameterstest.cpp b/vdslib/src/tests/container/parameterstest.cpp index c54d8ae66da..95a29fb97be 100644 --- a/vdslib/src/tests/container/parameterstest.cpp +++ b/vdslib/src/tests/container/parameterstest.cpp @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/vdslib/container/parameters.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/gtest/gtest.h> -using document::DocumentTypeRepo; +using vespalib::GrowableByteBuffer; +using document::ByteBuffer; using namespace vdslib; TEST(ParametersTest, test_parameters) @@ -15,11 +17,12 @@ TEST(ParametersTest, test_parameters) par.set("number", 6); par.set("int64_t", INT64_C(8589934590)); par.set("double", 0.25); - std::unique_ptr<document::ByteBuffer> buffer(par.serialize()); - buffer->flip(); - DocumentTypeRepo repo; - Parameters par2(repo, *buffer); + GrowableByteBuffer buffer; + par.serialize(buffer); + + ByteBuffer bBuf(buffer.getBuffer(), buffer.position()); + Parameters par2(bBuf); EXPECT_EQ(vespalib::stringref("overture"), par2.get("fast")); EXPECT_EQ(vespalib::stringref("yahoo"), par2.get("overture")); @@ -35,4 +38,5 @@ TEST(ParametersTest, test_parameters) EXPECT_EQ(numberDefault, par2.get("nonexistingnumber", numberDefault)); EXPECT_EQ(int64Default, par2.get("nonexistingint64_t", int64Default)); EXPECT_EQ(doubleDefault, par2.get("nonexistingdouble", doubleDefault)); + } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.cpp b/vdslib/src/vespa/vdslib/container/documentsummary.cpp index bc8a0473ab3..f948bd64687 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.cpp +++ b/vdslib/src/vespa/vdslib/container/documentsummary.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentsummary.h" +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <algorithm> namespace vdslib { @@ -21,7 +23,7 @@ DocumentSummary::DocumentSummary(document::ByteBuffer& buf) : deserialize(buf); } -DocumentSummary::~DocumentSummary() {} +DocumentSummary::~DocumentSummary() = default; void DocumentSummary::deserialize(document::ByteBuffer& buf) { @@ -47,18 +49,18 @@ void DocumentSummary::deserialize(document::ByteBuffer& buf) } } -void DocumentSummary::serialize(document::ByteBuffer& buf) const +void DocumentSummary::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putIntNetwork(0); // Just serialize dummy 4 byte field, to avoid versioning. - buf.putIntNetwork(_summary.size()); + buf.putInt(0); // Just serialize dummy 4 byte field, to avoid versioning. + buf.putInt(_summary.size()); if ( ! _summary.empty() ) { - buf.putIntNetwork(getSummarySize()); + buf.putInt(getSummarySize()); for (size_t i(0), m(_summary.size()); i < m; i++) { Summary s(_summary[i]); buf.putBytes(s.getDocId(_summaryBuffer->c_str()), s.getTotalSize()); } for (size_t i(0), m(_summary.size()); i < m; i++) { - buf.putIntNetwork(_summary[i].getSummarySize()); + buf.putInt(_summary[i].getSummarySize()); } } } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.h b/vdslib/src/vespa/vdslib/container/documentsummary.h index f04c1fa06bf..375546920ec 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.h +++ b/vdslib/src/vespa/vdslib/container/documentsummary.h @@ -2,9 +2,10 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { class DocumentSummary { @@ -29,7 +30,7 @@ public: void sort(); void deserialize(document::ByteBuffer& buf); - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: class Summary { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 4358c42ff29..9c843df1caa 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -1,20 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "parameters.hpp" +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/xmlstream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <ostream> using namespace vdslib; Parameters::Parameters() = default; -Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +Parameters::Parameters(document::ByteBuffer& buffer) : _parameters() { - deserialize(repo, buffer); + deserialize(buffer); } Parameters::~Parameters() = default; @@ -28,20 +30,19 @@ size_t Parameters::getSerializedSize() const return mysize; } -void Parameters::onSerialize(document::ByteBuffer& buffer) const +void Parameters::serialize(vespalib::GrowableByteBuffer& buffer) const { - buffer.putIntNetwork(_parameters.size()); + buffer.putInt(_parameters.size()); for (const auto & entry : _parameters) { - buffer.putIntNetwork(entry.first.size()); + buffer.putInt(entry.first.size()); buffer.putBytes(entry.first.c_str(), entry.first.size()); - buffer.putIntNetwork(entry.second.size()); + buffer.putInt(entry.second.size()); buffer.putBytes(entry.second.c_str(), entry.second.size()); } } -void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +void Parameters::deserialize(document::ByteBuffer& buffer) { - (void) repo; _parameters.clear(); int32_t mysize; buffer.getIntNetwork(mysize); @@ -88,11 +89,6 @@ Parameters::operator==(const Parameters &other) const return true; } -Parameters* Parameters::clone() const -{ - return new Parameters(*this); -} - vespalib::stringref Parameters::get(vespalib::stringref id, vespalib::stringref def) const { ParametersMap::const_iterator it = _parameters.find(id); @@ -130,7 +126,7 @@ void Parameters::print(std::ostream& out, bool verbose, const std::string& inden out << ")"; } -std::string Parameters::toString() const +vespalib::string Parameters::toString() const { vespalib::string ret; for (const auto & entry : _parameters) { diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h index f3ea0543546..61649b29bbe 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.h +++ b/vdslib/src/vespa/vdslib/container/parameters.h @@ -14,18 +14,15 @@ #pragma once -#include <vespa/document/util/serializable.h> #include <vespa/document/util/xmlserializable.h> #include <vespa/vespalib/stllike/hash_map.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class GrowableByteBuffer; } +namespace document { class ByteBuffer; } namespace vdslib { -class Parameters : public document::Deserializable, - public document::XmlSerializable { +class Parameters : public document::XmlSerializable { public: typedef vespalib::stringref KeyT; class Value : public vespalib::string @@ -42,27 +39,25 @@ public: private: ParametersMap _parameters; - void onSerialize(document::ByteBuffer& buffer) const override; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; void printXml(document::XmlOutputStream& xos) const override; public: Parameters(); - Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer); - virtual ~Parameters(); + Parameters(document::ByteBuffer& buffer); + ~Parameters(); bool operator==(const Parameters &other) const; - Parameters* clone() const override; + size_t getSerializedSize() const; - size_t getSerializedSize() const override; - - bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } - unsigned int size() const { return _parameters.size(); } + bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } + unsigned int size() const { return _parameters.size(); } bool lookup(KeyT id, ValueRef & v) const; void set(KeyT id, const void * v, size_t sz) { _parameters[id] = Value(v, sz); } void print(std::ostream& out, bool verbose, const std::string& indent) const; + void serialize(vespalib::GrowableByteBuffer& buffer) const; + void deserialize(document::ByteBuffer& buffer); // Disallow ParametersMap::const_iterator begin() const { return _parameters.begin(); } @@ -92,7 +87,7 @@ public: template<typename T> T get(KeyT id, T def) const; - std::string toString() const; + vespalib::string toString() const; }; } // vdslib diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index 73b19a2f8a2..20cc53e2de9 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchresult.h" +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <algorithm> namespace vdslib { @@ -25,21 +27,21 @@ void AggregatorList::deserialize(document::ByteBuffer & buf) } } -void AggregatorList::serialize(document::ByteBuffer & buf) const +void AggregatorList::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(size()); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - buf.putIntNetwork(it->first); - buf.putIntNetwork(it->second.size()); - buf.putBytes(it->second, it->second.size()); + buf.putInt(size()); + for (const auto & entry : *this) { + buf.putInt(entry.first); + buf.putInt(entry.second.size()); + buf.putBytes(entry.second, entry.second.size()); } } uint32_t AggregatorList::getSerializedSize() const { size_t sz(sizeof(uint32_t) * (1 + 2*size())); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - sz += it->second.size(); + for (const auto & entry : *this) { + sz += entry.second.size(); } return sz; } @@ -51,7 +53,7 @@ BlobContainer::BlobContainer(size_t reserve) : _offsets.push_back(0); } -BlobContainer::~BlobContainer() {} +BlobContainer::~BlobContainer() = default; size_t BlobContainer::append(const void * v, size_t sz) { @@ -84,11 +86,11 @@ void BlobContainer::deserialize(document::ByteBuffer & buf) buf.getBytes(_blob, getSize()); } -void BlobContainer::serialize(document::ByteBuffer & buf) const +void BlobContainer::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(getCount()); + buf.putInt(getCount()); for(size_t i(0), m(getCount()); i < m; i++) { - buf.putIntNetwork(getSize(i)); + buf.putInt(getSize(i)); } buf.putBytes(_blob, getSize()); } @@ -116,7 +118,7 @@ SearchResult::SearchResult(document::ByteBuffer & buf) : deserialize(buf); } -SearchResult::~SearchResult() {} +SearchResult::~SearchResult() = default; void SearchResult::deserialize(document::ByteBuffer & buf) { @@ -143,26 +145,26 @@ void SearchResult::deserialize(document::ByteBuffer & buf) _groupingList.deserialize(buf); } -void SearchResult::serialize(document::ByteBuffer & buf) const +void SearchResult::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(_totalHits); + buf.putInt(_totalHits); uint32_t hitCount = std::min(_hits.size(), _wantedHits); - buf.putIntNetwork(hitCount); + buf.putInt(hitCount); if (hitCount > 0) { uint32_t sz = getBufCount(); - buf.putIntNetwork(sz); + buf.putInt(sz); for (size_t i(0), m(hitCount); i < m; i++) { const char * s(_hits[i].getDocId(_docIdBuffer->c_str())); buf.putBytes(s, strlen(s)+1); } for (size_t i(0), m(hitCount); i < m; i++) { - buf.putDoubleNetwork(_hits[i].getRank()); + buf.putDouble(_hits[i].getRank()); } } uint32_t sortCount = std::min(_sortBlob.getCount(), _wantedHits); - buf.putIntNetwork(sortCount); + buf.putInt(sortCount); for (size_t i(0); i < sortCount; i++) { - buf.putIntNetwork(_sortBlob.getSize(_hits[i].getIndex())); + buf.putInt(_sortBlob.getSize(_hits[i].getIndex())); } for (size_t i(0); i < sortCount; i++) { size_t sz; diff --git a/vdslib/src/vespa/vdslib/container/searchresult.h b/vdslib/src/vespa/vdslib/container/searchresult.h index 081873e2989..fc893f6b5be 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.h +++ b/vdslib/src/vespa/vdslib/container/searchresult.h @@ -2,10 +2,11 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> #include <map> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { typedef std::map<size_t, vespalib::MallocPtr> IntBlobMapT; @@ -15,7 +16,7 @@ class AggregatorList : public IntBlobMapT public: void add(size_t id, const vespalib::MallocPtr & aggrBlob); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; }; @@ -31,7 +32,7 @@ public: size_t getSize(size_t index) const { return _offsets[index+1] - _offsets[index]; } const void * getBuf(size_t index) const { return _blob.c_str() + _offsets[index]; } void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const { return (1 + getCount()) * sizeof(uint32_t) + getSize(); } private: typedef vespalib::MallocPtr Blob; @@ -76,7 +77,7 @@ public: void sort(); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; private: class Hit { diff --git a/vdslib/src/vespa/vdslib/state/nodestate.cpp b/vdslib/src/vespa/vdslib/state/nodestate.cpp index 22854b7a571..41d42fd5c6f 100644 --- a/vdslib/src/vespa/vdslib/state/nodestate.cpp +++ b/vdslib/src/vespa/vdslib/state/nodestate.cpp @@ -17,8 +17,8 @@ namespace storage::lib { NodeState::NodeState(const NodeState &) = default; NodeState & NodeState::operator = (const NodeState &) = default; -NodeState::NodeState(NodeState &&) = default; -NodeState & NodeState::operator = (NodeState &&) = default; +NodeState::NodeState(NodeState &&) noexcept = default; +NodeState & NodeState::operator = (NodeState &&) noexcept = default; NodeState::~NodeState() { } NodeState::NodeState() diff --git a/vdslib/src/vespa/vdslib/state/nodestate.h b/vdslib/src/vespa/vdslib/state/nodestate.h index 74f58663650..6317cb3fa84 100644 --- a/vdslib/src/vespa/vdslib/state/nodestate.h +++ b/vdslib/src/vespa/vdslib/state/nodestate.h @@ -40,8 +40,8 @@ public: NodeState(); NodeState(const NodeState &); NodeState & operator = (const NodeState &); - NodeState(NodeState &&); - NodeState & operator = (NodeState &&); + NodeState(NodeState &&) noexcept; + NodeState & operator = (NodeState &&) noexcept; NodeState(const NodeType& nodeType, const State&, vespalib::stringref description = "", double capacity = 1.0, uint16_t reliability = 1); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java new file mode 100644 index 00000000000..49b10a37329 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java @@ -0,0 +1,51 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.api; + +import java.util.Objects; + +/** + * Represents an Athenz Access Token + * + * @author bjorncs + */ +public class AthenzAccessToken { + + public static final String HTTP_HEADER_NAME = "Authorization"; + + private static final String BEARER_TOKEN_PREFIX = "Bearer "; + + private final String value; + + public AthenzAccessToken(String value) { + this.value = stripBearerTokenPrefix(value); + } + + private static String stripBearerTokenPrefix(String rawValue) { + String stripped = rawValue.strip(); + String prefixRemoved = stripped.startsWith(BEARER_TOKEN_PREFIX) + ? stripped.substring(BEARER_TOKEN_PREFIX.length()).strip() + : stripped; + if (prefixRemoved.isBlank()) { + throw new IllegalArgumentException(String.format("Access token is blank: '%s'", prefixRemoved)); + } + return prefixRemoved; + } + + public String value() { return value; } + public String valueWithBearerPrefix() { return BEARER_TOKEN_PREFIX + value; } + + @Override public String toString() { return "AthenzAccessToken{value='" + value + "'}"; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthenzAccessToken that = (AthenzAccessToken) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java index 3a81e4a5e17..4e432768298 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java @@ -7,6 +7,8 @@ import java.util.Objects; * @author tokle */ public class AthenzRole { + private static final String ROLE_RESOURCE_PREFIX = "role."; + private final AthenzDomain domain; private final String roleName; @@ -20,6 +22,19 @@ public class AthenzRole { this.roleName = roleName; } + public static AthenzRole fromResourceNameString(String string) { + return fromResourceName(AthenzResourceName.fromString(string)); + } + + public static AthenzRole fromResourceName(AthenzResourceName resourceName) { + String entityName = resourceName.getEntityName(); + if (!entityName.startsWith(ROLE_RESOURCE_PREFIX)) { + throw new IllegalArgumentException("Not a valid role: " + resourceName.toResourceNameString()); + } + String roleName = entityName.substring(ROLE_RESOURCE_PREFIX.length()); + return new AthenzRole(resourceName.getDomain(), roleName); + } + public AthenzDomain domain() { return domain; } @@ -28,6 +43,10 @@ public class AthenzRole { return roleName; } + public String toResourceNameString() { return toResourceName().toResourceNameString(); } + + public AthenzResourceName toResourceName() { return new AthenzResourceName(domain, ROLE_RESOURCE_PREFIX + roleName); } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index eec578ddc26..c05213c8008 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -2,14 +2,17 @@ package com.yahoo.vespa.athenz.client.zts; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity; @@ -36,6 +39,7 @@ import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -147,6 +151,33 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override + public AthenzAccessToken getAccessToken(AthenzDomain domain) { + return this.getAccessTokenImpl(List.of(new AthenzResourceName(domain, "domain"))); + } + + @Override + public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) { + List<AthenzResourceName> athenzResourceNames = athenzRole.stream() + .map(AthenzRole::toResourceName) + .collect(toList()); + return this.getAccessTokenImpl(athenzResourceNames); + } + + private AthenzAccessToken getAccessTokenImpl(List<AthenzResourceName> resources) { + URI uri = ztsUrl.resolve("oauth2/token"); + RequestBuilder requestBuilder = RequestBuilder.post(uri) + .addHeader("Content-Type", "application/x-www-form-urlencoded") + .addParameter("grant_type", "client_credentials") + .addParameter("scope", resources.stream().map(AthenzResourceName::toResourceNameString).collect(Collectors.joining(" "))); + + HttpUriRequest request = requestBuilder.build(); + return execute(request, response -> { + AccessTokenResponseEntity accessTokenResponseEntity = readEntity(response, AccessTokenResponseEntity.class); + return accessTokenResponseEntity.accessToken(); + }); + } + + @Override public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) { RoleCertificateRequestEntity requestEntity = new RoleCertificateRequestEntity(csr, expiry); URI uri = ztsUrl.resolve(String.format("domain/%s/role/%s/token", role.domain().getName(), role.roleName())); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index c09ad8f48a0..baeb4271905 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.client.zts; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; @@ -78,6 +79,22 @@ public interface ZtsClient extends AutoCloseable { ZToken getRoleToken(AthenzRole athenzRole); /** + * Fetch an access token for the target domain + * + * @param domain Target domain + * @return An Athenz access token + */ + AthenzAccessToken getAccessToken(AthenzDomain domain); + + /** + * Fetch an access token for the target roles + * + * @param athenzRole List of athenz roles to get access token for + * @return An Athenz access token + */ + AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole); + + /** * Fetch role certificate for the target domain and role * * @param role Target role diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java new file mode 100644 index 00000000000..edd423b52bd --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; + +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AccessTokenResponseEntity { + private final AthenzAccessToken accessToken; + private final Instant expiryTime; + private final List<AthenzRole> roles; + + public AccessTokenResponseEntity( + @JsonProperty("access_token") String accessToken, + @JsonProperty("expires_in") int expiresIn, + @JsonProperty("scope") String roles) { + + this.accessToken = new AthenzAccessToken(accessToken); + // We do not know from when, so best we can do is assume now ... + this.expiryTime = Instant.now().plusSeconds(expiresIn); + this.roles = Stream.of(roles.split(" ")) + .map(AthenzResourceName::fromString) + .map(AthenzRole::fromResourceName) + .collect(Collectors.toList()); + } + + public AthenzAccessToken accessToken() { + return accessToken; + } + + public Instant expiryTime() { + return expiryTime; + } + + public List<AthenzRole> roles() { + return roles; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index bea9af458b4..5d6f0e3ce16 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -16,6 +16,7 @@ import com.yahoo.security.KeyStoreType; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.tls.MutableX509KeyManager; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; @@ -84,6 +85,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; + private final LoadingCache<AthenzDomain, AthenzAccessToken> domainSpecificAccessTokenCache; + private final LoadingCache<List<AthenzRole>, AthenzAccessToken> roleSpecificAccessTokenCache; private final CsrGenerator csrGenerator; @Inject @@ -116,6 +119,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext); roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); + domainSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); + roleSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName()); this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore); registerInstance(); @@ -199,6 +204,16 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override + public String getAccessToken(String domain) { + return null; + } + + @Override + public String getAccessToken(String domain, List<String> roles) { + return null; + } + + @Override public PrivateKey getPrivateKey() { return credentials.getKeyPair().getPrivate(); } @@ -240,6 +255,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } + private AthenzAccessToken createAccessToken(AthenzDomain domain) { + try (ZtsClient client = createZtsClient()) { + return client.getAccessToken(domain); + } + } + + private AthenzAccessToken createAccessToken(List<AthenzRole> roles) { + try (ZtsClient client = createZtsClient()) { + return client.getAccessToken(roles); + } + } + private DefaultZtsClient createZtsClient() { return new DefaultZtsClient(ztsEndpoint, getIdentitySslContext()); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java index 33e5552eaf6..bec21a5b25f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java @@ -17,20 +17,8 @@ import static com.yahoo.security.SubjectAlternativeName.Type.RFC822_NAME; */ public class AthenzX509CertificateUtils { - private static final String COMMON_NAME_ROLE_DELIMITER = ":role."; - private AthenzX509CertificateUtils() {} - public static boolean isAthenzRoleCertificate(X509Certificate certificate) { - return isAthenzIssuedCertificate(certificate) && - com.yahoo.security.X509CertificateUtils.getSubjectCommonNames(certificate).get(0).contains(COMMON_NAME_ROLE_DELIMITER); - } - - public static boolean isAthenzIssuedCertificate(X509Certificate certificate) { - return com.yahoo.security.X509CertificateUtils.getIssuerCommonNames(certificate).stream() - .anyMatch(cn -> cn.equalsIgnoreCase("Yahoo Athenz CA") || cn.equalsIgnoreCase("Athenz AWS CA")); - } - public static AthenzIdentity getIdentityFromRoleCertificate(X509Certificate certificate) { List<com.yahoo.security.SubjectAlternativeName> sans = com.yahoo.security.X509CertificateUtils.getSubjectAlternativeNames(certificate); return sans.stream() @@ -43,10 +31,7 @@ public class AthenzX509CertificateUtils { public static AthenzRole getRolesFromRoleCertificate(X509Certificate certificate) { String commonName = com.yahoo.security.X509CertificateUtils.getSubjectCommonNames(certificate).get(0); - int delimiterIndex = commonName.indexOf(COMMON_NAME_ROLE_DELIMITER); - String domain = commonName.substring(0, delimiterIndex); - String roleName = commonName.substring(delimiterIndex + COMMON_NAME_ROLE_DELIMITER.length()); - return new AthenzRole(domain, roleName); + return AthenzRole.fromResourceNameString(commonName); } private static AthenzIdentity getIdentityFromSanEmail(String email) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java index faf05011af9..28001e8e8d2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java @@ -2,45 +2,87 @@ package com.yahoo.vespa.athenz.zpe; import com.yahoo.athenz.zpe.AuthZpeClient.AccessCheckStatus; +import com.yahoo.vespa.athenz.api.AthenzRole; import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; /** * The various types of access control results. * * @author bjorncs */ -public enum AuthorizationResult { - ALLOW(AccessCheckStatus.ALLOW), - DENY(AccessCheckStatus.DENY), - DENY_NO_MATCH(AccessCheckStatus.DENY_NO_MATCH), - DENY_ROLETOKEN_EXPIRED(AccessCheckStatus.DENY_ROLETOKEN_EXPIRED), - DENY_ROLETOKEN_INVALID(AccessCheckStatus.DENY_ROLETOKEN_INVALID), - DENY_DOMAIN_MISMATCH(AccessCheckStatus.DENY_DOMAIN_MISMATCH), - DENY_DOMAIN_NOT_FOUND(AccessCheckStatus.DENY_DOMAIN_NOT_FOUND), - DENY_DOMAIN_EXPIRED(AccessCheckStatus.DENY_DOMAIN_EXPIRED), - DENY_DOMAIN_EMPTY(AccessCheckStatus.DENY_DOMAIN_EMPTY), - DENY_INVALID_PARAMETERS(AccessCheckStatus.DENY_INVALID_PARAMETERS), - DENY_CERT_MISMATCH_ISSUER(AccessCheckStatus.DENY_CERT_MISMATCH_ISSUER), - DENY_CERT_MISSING_SUBJECT(AccessCheckStatus.DENY_CERT_MISSING_SUBJECT), - DENY_CERT_MISSING_DOMAIN(AccessCheckStatus.DENY_CERT_MISSING_DOMAIN), - DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME); - - private final AccessCheckStatus wrappedElement; - - AuthorizationResult(AccessCheckStatus wrappedElement) { - this.wrappedElement = wrappedElement; +public class AuthorizationResult { + + private final Type type; + private final AthenzRole matchedRole; + + public AuthorizationResult(Type type) { + this(type, null); + } + + public AuthorizationResult(Type type, AthenzRole matchedRole) { + this.type = type; + this.matchedRole = matchedRole; + } + + public Type type() { return type; } + public Optional<AthenzRole> matchedRole() { return Optional.ofNullable(matchedRole); } + + public enum Type { + ALLOW(AccessCheckStatus.ALLOW), + DENY(AccessCheckStatus.DENY), + DENY_NO_MATCH(AccessCheckStatus.DENY_NO_MATCH), + DENY_ROLETOKEN_EXPIRED(AccessCheckStatus.DENY_ROLETOKEN_EXPIRED), + DENY_ROLETOKEN_INVALID(AccessCheckStatus.DENY_ROLETOKEN_INVALID), + DENY_DOMAIN_MISMATCH(AccessCheckStatus.DENY_DOMAIN_MISMATCH), + DENY_DOMAIN_NOT_FOUND(AccessCheckStatus.DENY_DOMAIN_NOT_FOUND), + DENY_DOMAIN_EXPIRED(AccessCheckStatus.DENY_DOMAIN_EXPIRED), + DENY_DOMAIN_EMPTY(AccessCheckStatus.DENY_DOMAIN_EMPTY), + DENY_INVALID_PARAMETERS(AccessCheckStatus.DENY_INVALID_PARAMETERS), + DENY_CERT_MISMATCH_ISSUER(AccessCheckStatus.DENY_CERT_MISMATCH_ISSUER), + DENY_CERT_MISSING_SUBJECT(AccessCheckStatus.DENY_CERT_MISSING_SUBJECT), + DENY_CERT_MISSING_DOMAIN(AccessCheckStatus.DENY_CERT_MISSING_DOMAIN), + DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME); + + private final AccessCheckStatus wrappedElement; + + Type(AccessCheckStatus wrappedElement) { + this.wrappedElement = wrappedElement; + } + + public String getDescription() { + return wrappedElement.toString(); + } + + static Type fromAccessCheckStatus(AccessCheckStatus accessCheckStatus) { + return Arrays.stream(values()) + .filter(value -> value.wrappedElement == accessCheckStatus) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown status: " + accessCheckStatus)); + } } - public String getDescription() { - return wrappedElement.toString(); + @Override + public String toString() { + return "AuthorizationResult{" + + "type=" + type + + ", matchedRole=" + matchedRole + + '}'; } - static AuthorizationResult fromAccessCheckStatus(AccessCheckStatus accessCheckStatus) { - return Arrays.stream(values()) - .filter(value -> value.wrappedElement == accessCheckStatus) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unknown status: " + accessCheckStatus)); + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AuthorizationResult that = (AuthorizationResult) o; + return type == that.type && + Objects.equals(matchedRole, that.matchedRole); } + @Override + public int hashCode() { + return Objects.hash(type, matchedRole); + } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java index 29044111ada..47ae45a69ca 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.athenz.zpe; import com.yahoo.athenz.zpe.AuthZpeClient; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.zpe.AuthorizationResult.Type; import java.security.cert.X509Certificate; @@ -21,14 +24,41 @@ public class DefaultZpe implements Zpe { @Override public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) { - return AuthorizationResult.fromAccessCheckStatus( - AuthZpeClient.allowAccess(roleToken.getRawToken(), resourceName.toResourceNameString(), action)); + StringBuilder returnedMatchedRole = new StringBuilder(); + AuthZpeClient.AccessCheckStatus rawResult = + AuthZpeClient.allowAccess(roleToken.getRawToken(), resourceName.toResourceNameString(), action, returnedMatchedRole); + return createResult(returnedMatchedRole, rawResult, resourceName); } @Override public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) { - return AuthorizationResult.fromAccessCheckStatus( - AuthZpeClient.allowAccess(roleCertificate, resourceName.toResourceNameString(), action)); + StringBuilder returnedMatchedRole = new StringBuilder(); + AuthZpeClient.AccessCheckStatus rawResult = + AuthZpeClient.allowAccess(roleCertificate, resourceName.toResourceNameString(), action, returnedMatchedRole); + return createResult(returnedMatchedRole, rawResult, resourceName); + } + + @Override + public AuthorizationResult checkAccessAllowed( + AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) { + StringBuilder returnedMatchedRole = new StringBuilder(); + AuthZpeClient.AccessCheckStatus rawResult = + AuthZpeClient.allowAccess( + accessToken.value(), identityCertificate, /*certHash*/null, resourceName.toResourceNameString(), action, returnedMatchedRole); + return createResult(returnedMatchedRole, rawResult, resourceName); + } + + private static AuthorizationResult createResult( + StringBuilder matchedRole, AuthZpeClient.AccessCheckStatus rawResult, AthenzResourceName resourceName) { + return new AuthorizationResult(Type.fromAccessCheckStatus(rawResult), toRole(matchedRole, resourceName)); + } + + private static AthenzRole toRole(StringBuilder rawRole, AthenzResourceName resourceName) { + if (rawRole.length() == 0) { + return null; + } else { + return new AthenzRole(resourceName.getDomain(), rawRole.toString()); + } } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java index e22e27f1508..51e5ee4dbb1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.zpe; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.ZToken; @@ -14,4 +15,5 @@ import java.security.cert.X509Certificate; public interface Zpe { AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action); AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action); + AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action); } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index d95eddac57f..95b6528bd77 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -465,7 +465,8 @@ public class DocumentGenMojo extends AbstractMojo { exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); - exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(),out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); + exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), + docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); exportCopyConstructor(className, out, 1, true); exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true); @@ -627,10 +628,21 @@ public class DocumentGenMojo extends AbstractMojo { } out.write(ind(ind) + "ret.addFieldSets(fieldSets);\n"); } + private static void exportImportedFields(Set<String> importedFieldNames, Writer out, int ind) throws IOException { + out.write(ind(ind) + "java.util.Set<java.lang.String> importedFieldNames = new java.util.HashSet<>();\n"); + for (String importedField : importedFieldNames) { + out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n"); + } + } private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets, - Writer out, int ind, String methodName, String retType) throws IOException { + Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException { out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n"); - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + if (importedFieldNames != null) { + exportImportedFields(importedFieldNames, out, ind + 1); + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); + } else { + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + } for (Field f : fields) { if (f.getDataType().equals(DataType.STRING)) { addExtendedStringField(className, f, out, ind + 1); @@ -783,7 +795,7 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind+2)+"super("+structClassName+".type);\n" + ind(ind+1)+"}\n\n"); exportCopyConstructor(structClassName, out, ind+1, false); - exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); exportAssign(structType, structClassName, out, ind+1); exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index fa463a77923..32cdbf9af5c 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -69,6 +69,7 @@ public class VespaDocumentOperation extends EvalFunc<String> { private static final String SIMPLE_OBJECT_FIELDS = "simple-object-fields"; private static final String CREATE_TENSOR_FIELDS = "create-tensor-fields"; private static final String EXCLUDE_FIELDS = "exclude-fields"; + private static final String TESTSET_CONDITION = "condition"; private static final String PARTIAL_UPDATE_ASSIGN = "assign"; @@ -162,7 +163,11 @@ public class VespaDocumentOperation extends EvalFunc<String> { if (op == Operation.UPDATE && createIfNonExistent) { writeField("create", true, DataType.BOOLEAN, g, properties, schema, op, 0); } - + String testSetConditionTemplate = properties.getProperty(TESTSET_CONDITION); + if (testSetConditionTemplate != null) { + String testSetCondition = TupleTools.toString(fields, testSetConditionTemplate); + writeField(TESTSET_CONDITION, testSetCondition, DataType.CHARARRAY, g, properties, schema, op, 0); + } if (op != Operation.REMOVE) { writeField("fields", fields, DataType.MAP, g, properties, schema, op, 0); } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 7d0fe72fc64..3c6805019b8 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -47,6 +47,19 @@ public class VespaDocumentOperationTest { assertEquals(3, fields.get("value").get("assign").getIntValue()); } + @Test + public void requireThatUDFSupportsConditionalUpdateAssign() throws IOException { + String json = getDocumentOperationJson("docid=id:<application>:metrics::<name>-<date>", "operation=update", "condition=clicks < <value>"); + ObjectMapper m = new ObjectMapper(); + JsonNode root = m.readTree(json); + JsonNode fields = root.path("fields"); + + assertEquals("id:testapp:metrics::clicks-20160112", root.get("update").getTextValue()); + assertEquals("clicks < 3", root.get("condition").getTextValue()); + assertEquals("testapp", fields.get("application").get("assign").getTextValue()); + assertEquals("clicks", fields.get("name").get("assign").getTextValue()); + assertEquals(3, fields.get("value").get("assign").getIntValue()); + } @Test public void requireThatUDFSupportsCreateIfNonExistent() throws IOException { diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index 275d9f97dcf..ed079442440 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -61,10 +61,11 @@ public class TestRunnerHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - if (path.equals("/tester/v1/log")) { - return new SlimeJsonResponse(toSlime(testRunner.getLog(request.hasProperty("after") - ? Long.parseLong(request.getProperty("after")) - : -1))); + // TODO: Migrate to /tester/v1/log when /tester/v1/log2 is not in use anymore (and remove /tester/v1/log2) + if (path.equals("/tester/v1/log") || path.equals("/tester/v1/log2")) { + return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after") + ? Long.parseLong(request.getProperty("after")) + : -1))); } else if (path.equals("/tester/v1/status")) { log.info("Responding with status " + testRunner.getStatus()); return new Response(testRunner.getStatus().name()); @@ -72,7 +73,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath()); } - private HttpResponse handlePOST(HttpRequest request) throws IOException, InterruptedException { + private HttpResponse handlePOST(HttpRequest request) throws IOException { final String path = request.getUri().getPath(); if (path.startsWith("/tester/v1/run/")) { String type = lastElement(path); @@ -93,9 +94,15 @@ public class TestRunnerHandler extends LoggingRequestHandler { return path.substring(lastSlash + 1, path.length()); } - static Slime toSlime(Collection<LogRecord> log) { - Slime root = new Slime(); - Cursor recordArray = root.setArray(); + static Slime logToSlime(Collection<LogRecord> log) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor recordArray = root.setArray("logRecords"); + logArrayToSlime(recordArray, log); + return slime; + } + + static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) { log.forEach(record -> { Cursor recordObject = recordArray.addObject(); recordObject.setLong("id", record.getSequenceNumber()); @@ -109,7 +116,6 @@ public class TestRunnerHandler extends LoggingRequestHandler { } recordObject.setString("message", message); }); - return root; } public static String typeOf(Level level) { @@ -120,7 +126,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { : "error"; } - private class SlimeJsonResponse extends HttpResponse { + private static class SlimeJsonResponse extends HttpResponse { private final Slime slime; private SlimeJsonResponse(Slime slime) { diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java index 3f6fecc8249..77de009571b 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java @@ -1,7 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -21,18 +21,43 @@ public class TestRunnerHandlerTest { @Test public void logSerialization() throws IOException { - LogRecord record = new LogRecord(Level.INFO, "Hello."); - record.setSequenceNumber(1); - record.setInstant(Instant.ofEpochMilli(2)); - Exception exception = new RuntimeException(); - record.setThrown(exception); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - exception.printStackTrace(new PrintStream(buffer)); - String trace = buffer.toString() - .replaceAll("\n", "\\\\n") - .replaceAll("\t", "\\\\t"); - assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]", - new String(SlimeUtils.toJsonBytes(TestRunnerHandler.toSlime(Collections.singletonList(record))))); + Log log = new Log(); + LogRecord record = log.getLogRecord(); + String trace = log.getTrace(); + assertEquals("{\"logRecords\":[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]}", + new String(SlimeUtils.toJsonBytes(TestRunnerHandler.logToSlime(Collections.singletonList(record))))); + } + + private static class Log { + + private final LogRecord record; + private final String trace; + + public Log() { + Exception exception = new RuntimeException(); + record = createRecord(exception); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + exception.printStackTrace(new PrintStream(buffer)); + trace = buffer.toString() + .replaceAll("\n", "\\\\n") + .replaceAll("\t", "\\\\t"); + } + + LogRecord getLogRecord() { + return record; + } + + String getTrace() { + return trace; + } + + private static LogRecord createRecord(Exception exception) { + LogRecord record = new LogRecord(Level.INFO, "Hello."); + record.setSequenceNumber(1); + record.setInstant(Instant.ofEpochMilli(2)); + record.setThrown(exception); + return record; + } } } diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index b970c5bed60..ad77965fef6 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -23,7 +23,7 @@ consider_fallback () { : $1 already has value $oldvariablevalue elif [ -z "${2}" ]; then : proposed value "${2}" is empty - elif [ `expr match "$2" ".*'"` != 0 ]; then + elif [ `expr "$2" : ".*'"` != 0 ]; then : proposed value "${2}" contains a single-quote else eval "${1}='${2}'" diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java index aeddc762586..0661363477f 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -202,7 +202,7 @@ public class RestApiTest { // Get logs through some hackish fetch method. Logs is something the mocked backend write. String getLog() throws IOException { - // The mocked backend will throw a runtime exception wtih a log if delete is called three times.. + // The mocked backend will throw a runtime exception with a log if delete is called three times.. Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); HttpDelete delete = new HttpDelete(request.getUri()); doRest(delete); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java index f09b2ba2e50..a185b03f7af 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Metric; import java.util.Map; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.20 */ class DummyMetric implements Metric { diff --git a/vespaclient-container-plugin/src/test/rest-api-application/services.xml b/vespaclient-container-plugin/src/test/rest-api-application/services.xml index 346740bc815..ae1b87635a9 100644 --- a/vespaclient-container-plugin/src/test/rest-api-application/services.xml +++ b/vespaclient-container-plugin/src/test/rest-api-application/services.xml @@ -2,6 +2,8 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <container version="1.0" jetty="true"> + <accesslog type="disabled"/> + <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test"> <binding>http://*/document/v1/*</binding> </handler> diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 80e93977980..4d0b3a8ff2e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.slime.*; +package com.yahoo.slime; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -9,8 +7,7 @@ import java.nio.charset.StandardCharsets; import java.util.Optional; /** - * Extra utilities/operations on slime trees that we would like to have as part of slime in the future, but - * which resides here until we have a better place to put it. + * Extra utilities/operations on slime trees. * * @author Ulf Lilleengen */ @@ -20,12 +17,7 @@ public class SlimeUtils { if (from.type() != Type.OBJECT) { throw new IllegalArgumentException("Cannot copy object: " + from); } - from.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - setObjectEntry(inspector, name, to); - } - }); + from.traverse((ObjectTraverser) (name, inspector) -> setObjectEntry(inspector, name, to)); } @@ -61,13 +53,7 @@ public class SlimeUtils { } private static void copyArray(Inspector from, final Cursor to) { - from.traverse(new ArrayTraverser() { - @Override - public void entry(int i, Inspector inspector) { - addValue(inspector, to); - } - }); - + from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to)); } private static void addValue(Inspector from, Cursor to) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java index 4d3989b8782..bccd66acd31 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java @@ -230,7 +230,7 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY @Override public String toString() { - return toString(null); + return toString(ToStringContext.empty()); } public String toString(ToStringContext context) { diff --git a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java index ff7b640630e..8b13ee74aed 100644 --- a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; +package com.yahoo.slime; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; import com.yahoo.text.Utf8; import org.junit.Test; @@ -14,9 +12,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.8 */ public class SlimeUtilsTest { + @Test public void test_copying_slime_types_into_cursor() { Slime slime = new Slime(); @@ -79,4 +77,5 @@ public class SlimeUtilsTest { assertThat(slime.get().field("foo").asString(), is("foobie")); assertTrue(slime.get().field("bar").valid()); } + } diff --git a/vespalib/src/tests/exception_classes/mmap.cpp b/vespalib/src/tests/exception_classes/mmap.cpp index 2d4c4796473..81b5e0de30e 100644 --- a/vespalib/src/tests/exception_classes/mmap.cpp +++ b/vespalib/src/tests/exception_classes/mmap.cpp @@ -3,6 +3,7 @@ #include <vector> #include <cassert> #include <string.h> +#include <cstdlib> #include <sys/resource.h> using namespace vespalib::alloc; diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp index 62bad716597..e938e15f4e6 100644 --- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp +++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp @@ -17,6 +17,7 @@ #include <fcntl.h> using namespace vespalib; +using namespace vespalib::test; struct SocketPair { SocketHandle client; @@ -204,7 +205,9 @@ void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_ser SocketHandle &my_handle = is_server ? sockets.server : sockets.client; my_handle.set_blocking(false); SmartBuffer read_buffer(4096); - CryptoSocket::UP my_socket = engine.create_crypto_socket(std::move(my_handle), is_server); + CryptoSocket::UP my_socket = is_server + ? engine.create_server_crypto_socket(std::move(my_handle)) + : engine.create_client_crypto_socket(std::move(my_handle), local_spec); TEST_DO(verify_handshake(*my_socket)); drain(*my_socket, read_buffer); TEST_DO(verify_socket_io(*my_socket, read_buffer, is_server)); @@ -226,19 +229,19 @@ TEST_MT_FFF("require that encrypted async socket io works with XorCryptoEngine", } TEST_MT_FFF("require that encrypted async socket io works with TlsCryptoEngine", - 2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60)) + 2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(true)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(false)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp index 88146cd4fb2..08893c9273b 100644 --- a/vespalib/src/tests/net/socket/socket_test.cpp +++ b/vespalib/src/tests/net/socket/socket_test.cpp @@ -204,14 +204,6 @@ TEST_MT_FF("require that basic unix domain socket io works (path)", 2, TEST_DO(verify_socket_io(is_server, socket)); } -TEST_MT_FF("require that basic unix domain socket io works (name)", 2, - ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) -{ - bool is_server = (thread_id == 0); - SocketHandle socket = connect_sockets(is_server, f1); - TEST_DO(verify_socket_io(is_server, socket)); -} - TEST_MT_FF("require that server accept can be interrupted", 2, ServerSocket("tcp/0"), TimeBomb(60)) { bool is_server = (thread_id == 0); if (is_server) { @@ -279,6 +271,15 @@ TEST("require that a server socket will remove an old socket file if it cannot b EXPECT_TRUE(!is_socket("my_socket")); } +#ifdef __linux__ +TEST_MT_FF("require that basic unix domain socket io works (name)", 2, + ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60)) +{ + bool is_server = (thread_id == 0); + SocketHandle socket = connect_sockets(is_server, f1); + TEST_DO(verify_socket_io(is_server, socket)); +} + TEST("require that two server sockets cannot have the same abstract unix domain socket name") { vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid())); ServerSocket server1(spec); @@ -313,6 +314,7 @@ TEST_MT_FFF("require that abstract and file-based unix domain sockets are not in SocketHandle socket = connect_sockets(is_server, server_socket); TEST_DO(verify_socket_io(is_server, socket)); } +#endif TEST("require that sockets can be set blocking and non-blocking") { SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); diff --git a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp index 0c2b92bbb53..f2da6b70bf3 100644 --- a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp +++ b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp @@ -123,4 +123,8 @@ TEST("require that replace_host gives invalid spec when used with less than 2 ho TEST_DO(verify_invalid(SocketSpec("ipc/name:my_socket").replace_host("foo"))); } +TEST("require that invalid socket spec is not valid") { + EXPECT_FALSE(SocketSpec::invalid.valid()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp index 76ecd9453b6..56767051dad 100644 --- a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp +++ b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp @@ -17,6 +17,7 @@ #include <fcntl.h> using namespace vespalib; +using namespace vespalib::test; struct SocketPair { SocketHandle client; @@ -97,7 +98,9 @@ void verify_socket_io(SyncCryptoSocket &socket, bool is_server) { void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_server) { SocketHandle &my_handle = is_server ? sockets.server : sockets.client; my_handle.set_blocking(false); - SyncCryptoSocket::UP my_socket = SyncCryptoSocket::create(engine, std::move(my_handle), is_server); + SyncCryptoSocket::UP my_socket = is_server + ? SyncCryptoSocket::create_server(engine, std::move(my_handle)) + : SyncCryptoSocket::create_client(engine, std::move(my_handle), local_spec); ASSERT_TRUE(my_socket); TEST_DO(verify_socket_io(*my_socket, is_server)); TEST_DO(verify_graceful_shutdown(*my_socket, is_server)); @@ -118,19 +121,19 @@ TEST_MT_FFF("require that encrypted sync socket io works with XorCryptoEngine", } TEST_MT_FFF("require that encrypted sync socket io works with TlsCryptoEngine", - 2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60)) + 2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(true)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(false)", - 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60)) + 2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60)) { TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0))); } diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp index e54700306fe..0bd029c0c3a 100644 --- a/vespalib/src/tests/portal/portal_test.cpp +++ b/vespalib/src/tests/portal/portal_test.cpp @@ -14,13 +14,14 @@ #include <vespa/vespalib/util/latch.h> using namespace vespalib; +using namespace vespalib::test; //----------------------------------------------------------------------------- vespalib::string do_http(int port, CryptoEngine::SP crypto, const vespalib::string &method, const vespalib::string &uri, bool send_host = true) { auto socket = SocketSpec::from_port(port).client_address().connect(); ASSERT_TRUE(socket.valid()); - auto conn = SyncCryptoSocket::create(*crypto, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto, std::move(socket), local_spec); vespalib::string http_req = vespalib::make_string("%s %s HTTP/1.1\r\n" "My-Header: my value\r\n" "%s" @@ -75,7 +76,7 @@ Encryption::~Encryption() = default; auto null_crypto() { return std::make_shared<NullCryptoEngine>(); } auto xor_crypto() { return std::make_shared<XorCryptoEngine>(); } -auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()); } +auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()); } auto maybe_tls_crypto(bool client_tls) { return std::make_shared<MaybeTlsCryptoEngine>(tls_crypto(), client_tls); } std::vector<Encryption> crypto_list = {{"no encryption", null_crypto()}, @@ -260,18 +261,18 @@ TEST("require that connection errors do not block shutdown by leaking resources" auto bound = portal->bind("/test", handler); { // close before sending anything auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); } { // send partial request then close connection auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); vespalib::string req = "GET /test HTTP/1.1\r\n" "Host: local"; ASSERT_EQUAL(conn->write(req.data(), req.size()), ssize_t(req.size())); } { // send request then close without reading response auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect(); - auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false); + auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec); vespalib::string req = "GET /test HTTP/1.1\r\n" "Host: localhost\r\n" "\r\n"; diff --git a/vespalib/src/tests/stllike/string_test.cpp b/vespalib/src/tests/stllike/string_test.cpp index 885c13172b1..fcce1386ebc 100644 --- a/vespalib/src/tests/stllike/string_test.cpp +++ b/vespalib/src/tests/stllike/string_test.cpp @@ -60,6 +60,45 @@ TEST("test self assignment of big string") { EXPECT_EQUAL(text, s); } +void verify_move_constructor(string org) { + string copy(org); + EXPECT_EQUAL(org, copy); + string moved_into(std::move(copy)); + EXPECT_EQUAL(org, moved_into); + EXPECT_NOT_EQUAL(org, copy); + EXPECT_EQUAL(string(), copy); +} + +void verify_move_operator(string org) { + string copy(org); + EXPECT_EQUAL(org, copy); + string moved_into_short("short movable string"); + EXPECT_LESS(moved_into_short.size(), string().capacity()); + EXPECT_NOT_EQUAL(org, moved_into_short); + moved_into_short = std::move(copy); + EXPECT_EQUAL(org, moved_into_short); + EXPECT_NOT_EQUAL(org, copy); + EXPECT_EQUAL(string(), copy); + + string moved_into_long("longer movable string than the 47 bytes that can be held in the short string optimization."); + EXPECT_GREATER(moved_into_long.size(), string().capacity()); + EXPECT_NOT_EQUAL(org, moved_into_long); + moved_into_long = std::move(moved_into_short); + EXPECT_EQUAL(org, moved_into_long); + EXPECT_NOT_EQUAL(org, moved_into_short); + EXPECT_EQUAL(string(), moved_into_short); +} + +void verify_move(string org) { + verify_move_constructor(org); + verify_move_operator(org); +} + +TEST("test move constructor") { + TEST_DO(verify_move("short string")); + TEST_DO(verify_move("longer string than the 47 bytes that can be held in the short string optimization.")); +} + TEST("testStringAlloc") { fprintf(stderr, "... testing allocations\n"); string a("abcde"); diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 758922aec6d..39c0f3f7482 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "databuffer.h" #include <algorithm> +#include <cstdio> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index d5fef404a43..4c289c04564 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -25,6 +25,7 @@ template <typename EntryT, typename RefT = EntryRefT<19> > class ArrayStore { public: + using ArrayRef = vespalib::ArrayRef<EntryT>; using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; using DataStoreType = DataStoreT<RefT>; using SmallArrayType = BufferType<EntryT>; @@ -82,6 +83,17 @@ public: return getLargeArray(internalRef); } } + + /** + * Returns a writeable reference to the given array. + * + * NOTE: Use with care if reader threads are accessing arrays at the same time. + * If so, replacing an element in the array should be an atomic operation. + */ + ArrayRef get_writable(EntryRef ref) { + return vespalib::unconstify(get(ref)); + } + void remove(EntryRef ref); ICompactionContext::UP compactWorst(bool compactMemory, bool compactAddressSpace); vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } diff --git a/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h new file mode 100644 index 00000000000..154c9669550 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "entryref.h" +#include <atomic> + +namespace search::datastore { + +/** + * A wrapper for std::atomic of type EntryRef that supports copy and move constructors and assignment operator, + * and uses Release-Acquire ordering for store and load. + * + * Use this class when entry refs are stored in data stores or rcu vectors, + * where copy and move constructors and assignment operator are needed when resizing underlying buffers. + * In this case synchronization between the writer thread and reader threads + * is handled as part of the buffer switch. + */ +class AtomicEntryRef { +private: + std::atomic<uint32_t> _ref; + +public: + AtomicEntryRef() noexcept : _ref() {} + explicit AtomicEntryRef(EntryRef ref) noexcept : _ref(ref.ref()) {} + AtomicEntryRef(const AtomicEntryRef& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {} + AtomicEntryRef(AtomicEntryRef&& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {} + AtomicEntryRef& operator=(const AtomicEntryRef& rhs) noexcept { + uint32_t ref = rhs._ref.load(std::memory_order_relaxed); + _ref.store(ref, std::memory_order_relaxed); + return *this; + } + + void store_release(EntryRef ref) noexcept { + _ref.store(ref.ref(), std::memory_order_release); + } + EntryRef load_acquire() const noexcept { + return EntryRef(_ref.load(std::memory_order_acquire)); + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp index 14abb93d8d0..39ea0d2d73b 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp @@ -17,4 +17,9 @@ AvxAccelrator::dotProduct(const double * af, const double * bf, size_t sz) const return avx::dotProductSelectAlignment<double, 32>(af, bf, sz); } +size_t +AvxAccelrator::populationCount(const uint64_t *a, size_t sz) const { + return helper::populationCount(a, sz); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx.h index ffbe0b8d27f..624531a9ca5 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx.h @@ -14,6 +14,7 @@ class AvxAccelrator : public Sse2Accelrator public: float dotProduct(const float * a, const float * b, size_t sz) const override; double dotProduct(const double * a, const double * b, size_t sz) const override; + size_t populationCount(const uint64_t *a, size_t sz) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp index 4c4e53e88db..ea8a3ead538 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp @@ -17,4 +17,9 @@ Avx2Accelrator::dotProduct(const double * af, const double * bf, size_t sz) cons return avx::dotProductSelectAlignment<double, 32>(af, bf, sz); } +size_t +Avx2Accelrator::populationCount(const uint64_t *a, size_t sz) const { + return helper::populationCount(a, sz); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h index f20068c6478..cf91bc81cfd 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h @@ -14,6 +14,7 @@ class Avx2Accelrator : public AvxAccelrator public: float dotProduct(const float * a, const float * b, size_t sz) const override; double dotProduct(const double * a, const double * b, size_t sz) const override; + size_t populationCount(const uint64_t *a, size_t sz) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp index 4d21c9358ec..1abf6b270cf 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp @@ -17,4 +17,9 @@ Avx512Accelrator::dotProduct(const double * af, const double * bf, size_t sz) co return avx::dotProductSelectAlignment<double, 64>(af, bf, sz); } +size_t +Avx512Accelrator::populationCount(const uint64_t *a, size_t sz) const { + return helper::populationCount(a, sz); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h index 5807aeeee57..eac8c96832b 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h @@ -14,6 +14,7 @@ class Avx512Accelrator : public Avx2Accelrator public: float dotProduct(const float * a, const float * b, size_t sz) const override; double dotProduct(const double * a, const double * b, size_t sz) const override; + size_t populationCount(const uint64_t *a, size_t sz) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index 2db7ebfd8fd..9e6a6d8817f 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -2,6 +2,7 @@ #pragma once +#include "private_helpers.hpp" #include <vespa/fastos/dynamiclibrary.h> #include <cstring> diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp index d70071525c6..b70ebb4051a 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "generic.h" +#include "private_helpers.hpp" namespace vespalib::hwaccelrated { @@ -124,4 +125,9 @@ GenericAccelrator::notBit(void * aOrg, size_t bytes) const } } +size_t +GenericAccelrator::populationCount(const uint64_t *a, size_t sz) const { + return helper::populationCount(a, sz); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h index f9aab3ae845..d76d0728bdd 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -22,6 +22,7 @@ public: void andBit(void * a, const void * b, size_t bytes) const override; void andNotBit(void * a, const void * b, size_t bytes) const override; void notBit(void * a, size_t bytes) const override; + size_t populationCount(const uint64_t *a, size_t sz) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp index aae277b48d8..061963f95a4 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp @@ -6,6 +6,8 @@ #include "avx.h" #include "avx2.h" #include "avx512.h" +#include <vespa/vespalib/util/memory.h> +#include <cstdio> #include <vespa/log/log.h> LOG_SETUP(".vespalib.hwaccelrated"); @@ -22,27 +24,27 @@ public: class GenericFactory :public Factory{ public: - IAccelrated::UP create() const override { return IAccelrated::UP(new GenericAccelrator()); } + IAccelrated::UP create() const override { return std::make_unique<GenericAccelrator>(); } }; class Sse2Factory :public Factory{ public: - IAccelrated::UP create() const override { return IAccelrated::UP(new Sse2Accelrator()); } + IAccelrated::UP create() const override { return std::make_unique<Sse2Accelrator>(); } }; class AvxFactory :public Factory{ public: - IAccelrated::UP create() const override { return IAccelrated::UP(new AvxAccelrator()); } + IAccelrated::UP create() const override { return std::make_unique<AvxAccelrator>(); } }; class Avx2Factory :public Factory{ public: - IAccelrated::UP create() const override { return IAccelrated::UP(new Avx2Accelrator()); } + IAccelrated::UP create() const override { return std::make_unique<Avx2Accelrator>(); } }; class Avx512Factory :public Factory{ public: - IAccelrated::UP create() const override { return IAccelrated::UP(new Avx512Accelrator()); } + IAccelrated::UP create() const override { return std::make_unique<Avx512Accelrator>(); } }; template<typename T> @@ -67,6 +69,23 @@ void verifyAccelrator(const IAccelrated & accel) delete [] b; } +void verifyPopulationCount(const IAccelrated & accel) +{ + const uint64_t words[7] = {0x123456789abcdef0L, // 32 + 0x0000000000000000L, // 0 + 0x8000000000000000L, // 1 + 0xdeadbeefbeefdeadUL, // 48 + 0x5555555555555555L, // 32 + 0x00000000000000001, // 1 + 0xffffffffffffffff}; // 64 + constexpr size_t expected = 32 + 0 + 1 + 48 + 32 + 1 + 64; + size_t hwComputedPopulationCount = accel.populationCount(words, VESPA_NELEMS(words)); + if (hwComputedPopulationCount != expected) { + fprintf(stderr, "Accelrator is not computing populationCount correctly.Expected %zu, computed %zu\n", expected, hwComputedPopulationCount); + LOG_ABORT("should not be reached"); + } +} + class RuntimeVerificator { public: @@ -79,7 +98,8 @@ RuntimeVerificator::RuntimeVerificator() verifyAccelrator<float>(generic); verifyAccelrator<double>(generic); verifyAccelrator<int32_t>(generic); - verifyAccelrator<int64_t>(generic); + verifyAccelrator<int64_t>(generic); + verifyPopulationCount(generic); IAccelrated::UP thisCpu(IAccelrated::getAccelrator()); verifyAccelrator<float>(*thisCpu); diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h index aae60279d06..4031169c44d 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h @@ -26,6 +26,7 @@ public: virtual void andBit(void * a, const void * b, size_t bytes) const = 0; virtual void andNotBit(void * a, const void * b, size_t bytes) const = 0; virtual void notBit(void * a, size_t bytes) const = 0; + virtual size_t populationCount(const uint64_t *a, size_t sz) const = 0; static IAccelrated::UP getAccelrator() __attribute__((noinline)); }; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp new file mode 100644 index 00000000000..8eba313d5f1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/optimized.h> + +namespace vespalib::hwaccelrated::helper { +namespace { + +inline size_t +populationCount(const uint64_t *a, size_t sz) { + size_t count(0); + size_t i(0); + for (; (i + 3) < sz; i += 4) { + count += Optimized::popCount(a[i + 0]) + + Optimized::popCount(a[i + 1]) + + Optimized::popCount(a[i + 2]) + + Optimized::popCount(a[i + 3]); + } + for (; i < sz; i++) { + count += Optimized::popCount(a[i]); + } + return count; +} + +} +}
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp index f135de52e5a..a0f584f8a9f 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "sse2.h" +#include "private_helpers.hpp" namespace vespalib::hwaccelrated { @@ -76,4 +77,9 @@ Sse2Accelrator::dotProduct(const double * af, const double * bf, size_t sz) cons return sum; } +size_t +Sse2Accelrator::populationCount(const uint64_t *a, size_t sz) const { + return helper::populationCount(a, sz); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h index a539aa44b03..d0fbefe5f03 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h @@ -14,6 +14,8 @@ class Sse2Accelrator : public GenericAccelrator public: float dotProduct(const float * a, const float * b, size_t sz) const override; double dotProduct(const double * a, const double * b, size_t sz) const override; + + size_t populationCount(const uint64_t *a, size_t sz) const override; }; } diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp index a52d4eeb690..a92b0e06bbe 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp @@ -250,16 +250,29 @@ CryptoEngine::get_default() } CryptoSocket::UP -NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +NullCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &) { - net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections(); + net::tls::ConnectionStatistics::get(false).inc_insecure_connections(); return std::make_unique<NullCryptoSocket>(std::move(socket)); } CryptoSocket::UP -XorCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +NullCryptoEngine::create_server_crypto_socket(SocketHandle socket) { - return std::make_unique<XorCryptoSocket>(std::move(socket), is_server); + net::tls::ConnectionStatistics::get(true).inc_insecure_connections(); + return std::make_unique<NullCryptoSocket>(std::move(socket)); +} + +CryptoSocket::UP +XorCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &) +{ + return std::make_unique<XorCryptoSocket>(std::move(socket), false); +} + +CryptoSocket::UP +XorCryptoEngine::create_server_crypto_socket(SocketHandle socket) +{ + return std::make_unique<XorCryptoSocket>(std::move(socket), true); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h index 1cb1305e039..4deacf9a6c7 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h @@ -9,6 +9,8 @@ namespace vespalib { +class SocketSpec; + /** * Component responsible for wrapping low-level sockets into * appropriate CryptoSocket instances. This is the top-level interface @@ -17,7 +19,8 @@ namespace vespalib { **/ struct CryptoEngine { using SP = std::shared_ptr<CryptoEngine>; - virtual CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) = 0; + virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; + virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0; virtual ~CryptoEngine(); static CryptoEngine::SP get_default(); }; @@ -26,7 +29,8 @@ struct CryptoEngine { * Crypto engine without encryption. **/ struct NullCryptoEngine : public CryptoEngine { - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; /** @@ -35,7 +39,8 @@ struct NullCryptoEngine : public CryptoEngine { * from TLS. **/ struct XorCryptoEngine : public CryptoEngine { - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h index c409e5a0e83..d51a97e2743 100644 --- a/vespalib/src/vespa/vespalib/net/crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h @@ -3,6 +3,7 @@ #pragma once #include <memory> +#include <cstdlib> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp index d1376ce1dd7..06682086670 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp +++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp @@ -41,6 +41,8 @@ SocketSpec::address(bool server) const return SocketAddress(); } +SocketSpec SocketSpec::invalid; + SocketSpec::SocketSpec(const vespalib::string &spec) : SocketSpec() { diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h index f28b14573ac..01af382d638 100644 --- a/vespalib/src/vespa/vespalib/net/socket_spec.h +++ b/vespalib/src/vespa/vespalib/net/socket_spec.h @@ -24,6 +24,7 @@ private: : _type(type), _node(node), _port(port) {} SocketAddress address(bool server) const; public: + static SocketSpec invalid; explicit SocketSpec(const vespalib::string &spec); vespalib::string spec() const; SocketSpec replace_host(const vespalib::string &new_host) const; diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp index 29388035bda..3aa2d3b0683 100644 --- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp @@ -29,6 +29,25 @@ void set_blocking(int fd) { } // namespace vespalib::<unnamed> +SyncCryptoSocket::UP +SyncCryptoSocket::create(CryptoSocket::UP socket) +{ + set_blocking(socket->get_fd()); + for (;;) { + switch (socket->handshake()) { + case CryptoSocket::HandshakeResult::FAIL: + return std::unique_ptr<SyncCryptoSocket>(nullptr); + case CryptoSocket::HandshakeResult::DONE: + return UP(new SyncCryptoSocket(std::move(socket))); + case CryptoSocket::HandshakeResult::NEED_READ: + case CryptoSocket::HandshakeResult::NEED_WRITE: + break; + case CryptoSocket::HandshakeResult::NEED_WORK: + socket->do_handshake_work(); + } + } +} + SyncCryptoSocket::~SyncCryptoSocket() = default; ssize_t @@ -90,23 +109,15 @@ SyncCryptoSocket::half_close() } SyncCryptoSocket::UP -SyncCryptoSocket::create(CryptoEngine &engine, SocketHandle socket, bool is_server) +SyncCryptoSocket::create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec) { - auto crypto_socket = engine.create_crypto_socket(std::move(socket), is_server); - set_blocking(crypto_socket->get_fd()); - for (;;) { - switch (crypto_socket->handshake()) { - case CryptoSocket::HandshakeResult::FAIL: - return std::unique_ptr<SyncCryptoSocket>(nullptr); - case CryptoSocket::HandshakeResult::DONE: - return UP(new SyncCryptoSocket(std::move(crypto_socket))); - case CryptoSocket::HandshakeResult::NEED_READ: - case CryptoSocket::HandshakeResult::NEED_WRITE: - break; - case CryptoSocket::HandshakeResult::NEED_WORK: - crypto_socket->do_handshake_work(); - } - } + return create(engine.create_client_crypto_socket(std::move(socket), spec)); +} + +SyncCryptoSocket::UP +SyncCryptoSocket::create_server(CryptoEngine &engine, SocketHandle socket) +{ + return create(engine.create_server_crypto_socket(std::move(socket))); } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h index 00d6cbca0db..36fcfe12ed9 100644 --- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h +++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h @@ -19,17 +19,20 @@ namespace vespalib { **/ class SyncCryptoSocket { +public: + using UP = std::unique_ptr<SyncCryptoSocket>; private: CryptoSocket::UP _socket; SmartBuffer _buffer; SyncCryptoSocket(CryptoSocket::UP socket) : _socket(std::move(socket)), _buffer(0) {} + static UP create(CryptoSocket::UP socket); public: - using UP = std::unique_ptr<SyncCryptoSocket>; ~SyncCryptoSocket(); ssize_t read(char *buf, size_t len); ssize_t write(const char *buf, size_t len); ssize_t half_close(); - static UP create(CryptoEngine &engine, SocketHandle socket, bool is_server); + static UP create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec); + static UP create_server(CryptoEngine &engine, SocketHandle socket); }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp index 5f20280e0e2..c425ab75ce8 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp @@ -91,13 +91,22 @@ AutoReloadingTlsCryptoEngine::EngineSP AutoReloadingTlsCryptoEngine::acquire_cur return _current_engine; } -CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) { - return acquire_current_engine()->create_crypto_socket(std::move(socket), is_server); +CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { + return acquire_current_engine()->create_client_crypto_socket(std::move(socket), spec); +} + +CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) { + return acquire_current_engine()->create_server_crypto_socket(std::move(socket)); +} + +std::unique_ptr<TlsCryptoSocket> +AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { + return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec); } std::unique_ptr<TlsCryptoSocket> -AutoReloadingTlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server) { - return acquire_current_engine()->create_tls_crypto_socket(std::move(socket), is_server); +AutoReloadingTlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) { + return acquire_current_engine()->create_tls_server_crypto_socket(std::move(socket)); } } diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h index 6287fdd4f63..e268cbc8f1a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h @@ -45,8 +45,10 @@ public: EngineSP acquire_current_engine() const; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; - std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; + std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; }; } diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp index 891f8cdab23..f7f0284bded 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp @@ -6,15 +6,19 @@ namespace vespalib { CryptoSocket::UP -MaybeTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) +MaybeTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) { - if (is_server) { - return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine); - } else if (_use_tls_when_client) { - return _tls_engine->create_crypto_socket(std::move(socket), false); + if (_use_tls_when_client) { + return _tls_engine->create_client_crypto_socket(std::move(socket), spec); } else { - return _null_engine->create_crypto_socket(std::move(socket), false); + return _null_engine->create_client_crypto_socket(std::move(socket), spec); } } +CryptoSocket::UP +MaybeTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) +{ + return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine); +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h index 29909fa115d..147a770bc8f 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h @@ -28,7 +28,8 @@ public: : _null_engine(std::make_shared<NullCryptoEngine>()), _tls_engine(std::move(tls_engine)), _use_tls_when_client(use_tls_when_client) {} - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp index 9af6703acbc..8ab6adad2e5 100644 --- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp @@ -50,7 +50,7 @@ public: } if (looksLikeTlsToMe(src.data)) { CryptoSocket::UP &self = _self; // need copy due to self destruction - auto tls_socket = _factory->create_tls_crypto_socket(std::move(_socket), true); + auto tls_socket = _factory->create_tls_server_crypto_socket(std::move(_socket)); tls_socket->inject_read_data(src.data, src.size); self = std::move(tls_socket); return self->handshake(); diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp index 58d99cc7108..d0475f3e88d 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp @@ -12,9 +12,17 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne } std::unique_ptr<TlsCryptoSocket> -TlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server) +TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &) { - auto mode = is_server ? net::tls::CryptoCodec::Mode::Server : net::tls::CryptoCodec::Mode::Client; + auto mode = net::tls::CryptoCodec::Mode::Client; + auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); + return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); +} + +std::unique_ptr<TlsCryptoSocket> +TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) +{ + auto mode = net::tls::CryptoCodec::Mode::Server; auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode); return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec)); } diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h index dc7d7eaf9ce..5e760cf5585 100644 --- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h +++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h @@ -11,7 +11,8 @@ namespace vespalib { class AbstractTlsCryptoEngine : public CryptoEngine { public: - virtual std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) = 0; + virtual std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0; + virtual std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) = 0; }; /** @@ -24,9 +25,13 @@ private: public: explicit TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce); - std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override; - CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override { - return create_tls_crypto_socket(std::move(socket), is_server); + std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override; + std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override; + CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override { + return create_tls_client_crypto_socket(std::move(socket), spec); + } + CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override { + return create_tls_server_crypto_socket(std::move(socket)); } std::shared_ptr<net::tls::TlsContext> tls_context() const noexcept { return _tls_ctx; }; diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index 3109295fbdc..daaea981b5a 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -141,7 +141,7 @@ public: size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } bool empty() const { return size() == 0; } - const char * c_str() const { return &_rbuf[0]; } + const char * data() const { return &_rbuf[0]; } const char * peek() const { return &_rbuf[_rp]; } size_t rp() const { return _rp; } nbostream & rp(size_t pos) { if (pos > _wp) fail(eof); _rp = pos; return *this; } diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index 47719ea4c69..a6d44348e5e 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -143,7 +143,7 @@ Portal::handle_accept(portal::HandleGuard guard, SocketHandle socket) { socket.set_blocking(false); socket.set_keepalive(true); - new HttpConnection(std::move(guard), _reactor, _crypto->create_crypto_socket(std::move(socket), true), + new HttpConnection(std::move(guard), _reactor, _crypto->create_server_crypto_socket(std::move(socket)), [this](HttpConnection *conn) { handle_http(conn); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 5eae4cea55e..3dc5de65285 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -22,14 +22,14 @@ public: typedef typename HashTable::const_iterator const_iterator; typedef typename HashTable::insert_result insert_result; public: - hash_map(hash_map &&) = default; - hash_map & operator = (hash_map &&) = default; + hash_map(hash_map &&) noexcept = default; + hash_map & operator = (hash_map &&) noexcept = default; hash_map(const hash_map &) = default; hash_map & operator = (const hash_map &) = default; hash_map(size_t reserveSize=0); hash_map(size_t reserveSize, H hasher, EQ equality); hash_map(std::initializer_list<value_type> input); - ~hash_map(); + ~hash_map() noexcept; iterator begin() { return _ht.begin(); } iterator end() { return _ht.end(); } const_iterator begin() const { return _ht.begin(); } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp index 311a256be76..08edcf3c837 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -25,7 +25,7 @@ hash_map<K, V, H, EQ, M>::hash_map(std::initializer_list<value_type> input) } template <typename K, typename V, typename H, typename EQ, typename M> -hash_map<K, V, H, EQ, M>::~hash_map() = default; +hash_map<K, V, H, EQ, M>::~hash_map() noexcept = default; template <typename K, typename V, typename H, typename EQ, typename M> void diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h index 919e5e8c47f..8d315ebfd07 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h @@ -19,8 +19,8 @@ public: typedef typename HashTable::const_iterator const_iterator; typedef typename HashTable::insert_result insert_result; public: - hash_set(hash_set &&) = default; - hash_set & operator = (hash_set &&) = default; + hash_set(hash_set &&) noexcept = default; + hash_set & operator = (hash_set &&) noexcept = default; hash_set(const hash_set &) = default; hash_set & operator = (const hash_set &) = default; hash_set(size_t reserveSize=0); diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index 4b425c4bded..ba9993c5a37 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -103,8 +103,8 @@ public: : _next(next), _node(node) {} hash_node(V &&node, next_t next=npos) : _next(next), _node(std::move(node)) {} - hash_node(hash_node &&) = default; - hash_node &operator=(hash_node &&) = default; + hash_node(hash_node &&) noexcept = default; + hash_node &operator=(hash_node &&) noexcept = default; hash_node(const hash_node &) = default; // These will not be created hash_node &operator=(const hash_node &) = default; // if V is non-copyable. bool operator == (const hash_node & rhs) const { @@ -221,8 +221,8 @@ public: typedef std::pair<iterator, bool> insert_result; public: - hashtable(hashtable &&) = default; - hashtable & operator = (hashtable &&) = default; + hashtable(hashtable &&) noexcept = default; + hashtable & operator = (hashtable &&) noexcept = default; hashtable(const hashtable &); hashtable & operator = (const hashtable &); hashtable(size_t reservedSpace); diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index e787ecc071f..ba37c57a0f9 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -177,6 +177,11 @@ public: small_string(const void * s, size_type sz) : _buf(_stack), _sz(sz) { init(s); } small_string(stringref s) : _buf(_stack), _sz(s.size()) { init(s.data()); } small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.data()); } + small_string(small_string && rhs) noexcept + : _sz(rhs.size()), _bufferSize(rhs._bufferSize) + { + move(std::move(rhs)); + } small_string(const small_string & rhs) noexcept : _buf(_stack), _sz(rhs.size()) { init(rhs.data()); } small_string(const small_string & rhs, size_type pos, size_type sz=npos) noexcept : _buf(_stack), _sz(std::min(sz, rhs.size()-pos)) @@ -200,7 +205,15 @@ public: free(buffer()); } } - small_string& operator= (const small_string &rhs) { + + small_string& operator= (small_string && rhs) noexcept { + reset(); + _sz = rhs._sz; + _bufferSize = rhs._bufferSize; + move(std::move(rhs)); + return *this; + } + small_string& operator= (const small_string &rhs) noexcept { return assign(rhs.data(), rhs.size()); } small_string & operator= (stringref rhs) { @@ -212,7 +225,7 @@ public: small_string& operator= (const std::string &rhs) { return operator= (stringref(rhs)); } - void swap(small_string & rhs) { + void swap(small_string & rhs) noexcept { std::swap(*this, rhs); } operator std::string () const { return std::string(c_str(), size()); } @@ -315,12 +328,12 @@ public: const char *found = (const char *)memchr(buf, c, _sz-start); return (found != NULL) ? (found - buffer()) : (size_type)npos; } - small_string & assign(const char * s) { return assign(s, strlen(s)); } - small_string & assign(const void * s, size_type sz); - small_string & assign(stringref s, size_type pos, size_type sz) { + small_string & assign(const char * s) noexcept { return assign(s, strlen(s)); } + small_string & assign(const void * s, size_type sz) noexcept; + small_string & assign(stringref s, size_type pos, size_type sz) noexcept { return assign(s.data() + pos, sz); } - small_string & assign(stringref rhs) { + small_string & assign(stringref rhs) noexcept { if (data() != rhs.data()) assign(rhs.data(), rhs.size()); return *this; } @@ -524,6 +537,20 @@ private: _reserveBytes(newBufferSize); } } + void move(small_string && rhs) { + if (rhs.isAllocated()) { + _buf = rhs._buf; + rhs._buf = rhs._stack; + rhs._sz = 0; + rhs._bufferSize = sizeof(rhs._stack); + rhs._stack[0] = 0; + } else { + _buf = _stack; + memcpy(_stack, rhs._stack, sizeof(_stack)); + rhs._sz = 0; + rhs._stack[0] = 0; + } + } typedef uint32_t isize_type; bool needAlloc(isize_type add) const { return (add + _sz + 1) > _bufferSize; } bool isAllocated() const { return _buf != _stack; } diff --git a/vespalib/src/vespa/vespalib/stllike/string.hpp b/vespalib/src/vespa/vespalib/stllike/string.hpp index bf98d82463b..f501a22adb4 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.hpp +++ b/vespalib/src/vespa/vespalib/stllike/string.hpp @@ -71,7 +71,7 @@ small_string<StackSize>::rfind(const char * s, size_type e) const { template <uint32_t StackSize> small_string<StackSize> & -small_string<StackSize>::assign(const void * s, size_type sz) { +small_string<StackSize>::assign(const void * s, size_type sz) noexcept { if (__builtin_expect(capacity() >= sz, true)) { char *buf = buffer(); memmove(buf, s, sz); diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp index c685bffc23e..dcd2ced8036 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp @@ -67,6 +67,8 @@ npxYSKVCyo3a/Vo33V8/H0WgOXioKEZJxA== namespace vespalib::test { +SocketSpec local_spec("tcp/localhost:123"); + vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() { return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem); } diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h index a1f1d5958f9..41e5d7cc86d 100644 --- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h +++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h @@ -2,11 +2,19 @@ #pragma once +#include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/transport_security_options.h> namespace vespalib::test { /** + * A socket spec representing "tcp/localhost:123". Used by unit tests + * performing hostname verification against the tls options created + * below. + **/ +extern SocketSpec local_spec; + +/** * Make security options allowing you to talk to yourself using * TLS. This is intended for testing purposes only. **/ diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index b52cace45a5..03ebc2807f9 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -59,13 +59,13 @@ public: bool resize_inplace(size_t newSize); Alloc(const Alloc &) = delete; Alloc & operator = (const Alloc &) = delete; - Alloc(Alloc && rhs) : + Alloc(Alloc && rhs) noexcept : _alloc(rhs._alloc), _allocator(rhs._allocator) { rhs.clear(); } - Alloc & operator=(Alloc && rhs) { + Alloc & operator=(Alloc && rhs) noexcept { if (this != & rhs) { if (_alloc.first != nullptr) { _allocator->free(_alloc); diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index ef39b449a45..1fdf3471fbb 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -87,12 +87,12 @@ public: Array(const Alloc & initial=Alloc::alloc()); Array(size_t sz, const Alloc & initial=Alloc::alloc()); Array(Alloc && buf, size_t sz); - Array(Array &&rhs); + Array(Array &&rhs) noexcept; Array(size_t sz, T value, const Alloc & initial=Alloc::alloc()); Array(const_iterator begin, const_iterator end, const Alloc & initial=Alloc::alloc()); Array(const Array & rhs); Array & operator =(const Array & rhs); - Array & operator =(Array && rhs); + Array & operator =(Array && rhs) noexcept; ~Array(); void swap(Array & rhs) { _array.swap(rhs._array); diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index cab9297aad9..8eb33af2a3a 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -74,7 +74,7 @@ Array<T> & Array<T>::operator =(const Array & rhs) } template <typename T> -Array<T> & Array<T>::operator =(Array && rhs) { +Array<T> & Array<T>::operator =(Array && rhs) noexcept { if (&rhs != this) { Array t(std::move(rhs)); swap(t); @@ -160,7 +160,7 @@ Array<T>::Array(Alloc && buf, size_t sz) : template <typename T> -Array<T>::Array(Array &&rhs) +Array<T>::Array(Array &&rhs) noexcept : _array(std::move(rhs._array)), _sz(rhs._sz) { diff --git a/vespalib/src/vespa/vespalib/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp index 8dfdac5ecc7..56533a77643 100644 --- a/vespalib/src/vespa/vespalib/util/compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/compressor.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/data/databuffer.h> +#include <stdexcept> using vespalib::alloc::Alloc; diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index 6804b096ee3..aa1441aa0bb 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -4,6 +4,7 @@ #include "traits.h" #include "arrayref.h" +#include <cstdlib> namespace vespalib { namespace stash { diff --git a/vespalog/src/vespa/log/log-assert.cpp b/vespalog/src/vespa/log/log-assert.cpp index 51ab7a019ad..28fac730189 100644 --- a/vespalog/src/vespa/log/log-assert.cpp +++ b/vespalog/src/vespa/log/log-assert.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "log.h" +#include <cstdio> LOG_SETUP(""); namespace ns_log { diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp index a43c6dd0416..635d982f529 100644 --- a/vespalog/src/vespa/log/log.cpp +++ b/vespalog/src/vespa/log/log.cpp @@ -233,11 +233,10 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, { const size_t sizeofEscapedPayload(msgSize*4+1); const size_t sizeofTotalMessage(sizeofEscapedPayload + 1000); - char * bigBuffer(new char[sizeofEscapedPayload+sizeofTotalMessage]); - char * escapedPayload(bigBuffer); - char * totalMessage(escapedPayload + sizeofEscapedPayload); + auto escapedPayload = std::make_unique<char[]>(sizeofEscapedPayload); + auto totalMessage = std::make_unique<char[]>(sizeofTotalMessage); - char *dst = escapedPayload; + char *dst = escapedPayload.get(); for (size_t i(0); (i < msgSize) && msg[i]; i++) { unsigned char c = static_cast<unsigned char>(msg[i]); if ((c >= 32) && (c != '\\') && (c != 127)) { @@ -274,14 +273,14 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, localtime_r(&secs, &tmbuf); char timebuf[100]; strftime(timebuf, 100, "%Y-%m-%d %H:%M:%S", &tmbuf); - snprintf(totalMessage, sizeofTotalMessage, + snprintf(totalMessage.get(), sizeofTotalMessage, "[%s.%06u] %d/%d (%s%s) %s: %s\n", timebuf, static_cast<unsigned int>(timestamp % 1000000), fakePid ? -1 : getpid(), tid, _prefix, _appendix, levelName(level), msg); } else if (level == debug || level == spam) { - snprintf(totalMessage, sizeofTotalMessage, + snprintf(totalMessage.get(), sizeofTotalMessage, "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", static_cast<unsigned int>(timestamp / 1000000), static_cast<unsigned int>(timestamp % 1000000), @@ -289,19 +288,18 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, _serviceName, _prefix, _appendix, levelName(level), file, line, _rcsId, - escapedPayload); + escapedPayload.get()); } else { - snprintf(totalMessage, sizeofTotalMessage, + snprintf(totalMessage.get(), sizeofTotalMessage, "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", static_cast<unsigned int>(timestamp / 1000000), static_cast<unsigned int>(timestamp % 1000000), _hostname, fakePid ? -1 : getpid(), tid, _serviceName, _prefix, - _appendix, levelName(level), escapedPayload); + _appendix, levelName(level), escapedPayload.get()); } - _target->write(totalMessage, strlen(totalMessage)); - delete [] bigBuffer; + _target->write(totalMessage.get(), strlen(totalMessage.get())); } const char * diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index ba2408240f3..b4dfc612890 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -7,6 +7,7 @@ #include <new> // for placement new #include <cstdlib> // for malloc #include <cstring> // for memset +#include <cstdarg> // for va_list #include <cinttypes> /** diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp index b016c13ab61..84f96fbbb3e 100644 --- a/vespamalloc/src/tests/overwrite/overwrite.cpp +++ b/vespamalloc/src/tests/overwrite/overwrite.cpp @@ -10,6 +10,13 @@ void check_ptr_real(void *ptr) void (*check_ptr)(void *ptr) = check_ptr_real; +void overwrite_memory_real(char *ptr, int offset) +{ + *(ptr + offset) = 0; +} + +void (*overwrite_memory)(char *ptr, int offset) = overwrite_memory_real; + class Test : public TestApp { public: @@ -63,16 +70,14 @@ void Test::testFillValue(char *a) void Test::verifyPreWriteDetection() { char * a = new char[8]; - *(a-1) = 0; - check_ptr(a); + overwrite_memory(a, -1); delete [] a; } void Test::verifyPostWriteDetection() { char * a = new char[8]; - a[8] = 0; - check_ptr(a); + overwrite_memory(a, 8); delete [] a; } diff --git a/vespamalloc/src/tests/test2/testgraph.cpp b/vespamalloc/src/tests/test2/testgraph.cpp index 9f25e45b7e9..ae0c468b185 100644 --- a/vespamalloc/src/tests/test2/testgraph.cpp +++ b/vespamalloc/src/tests/test2/testgraph.cpp @@ -3,6 +3,7 @@ #include <vespamalloc/util/callgraph.h> #include <vespamalloc/util/callstack.h> #include <vespamalloc/util/traceutil.h> +#include <string> using namespace vespamalloc; diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp index c954c1aae26..a3930c7cca1 100644 --- a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp +++ b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp @@ -15,12 +15,13 @@ size_t AllocPoolT<MemBlockPtrT>::_alwaysReuseLimit __attribute__((visibility("hi template <typename MemBlockPtrT> AllocPoolT<MemBlockPtrT>::AllocPoolT(DataSegment<MemBlockPtrT> & ds) : _chunkPool(NULL), + _scList(), _dataSegment(ds), _getChunks(0), _getChunksSum(0), - _allocChunkList(0) + _allocChunkList(0), + _stat() { - memset(_scList, 0, sizeof(_scList)); } template <typename MemBlockPtrT> diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index 3b1c98cd319..5f66d550174 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -54,8 +54,8 @@ FieldSearchSpec::FieldSearchSpec() : } FieldSearchSpec::~FieldSearchSpec() = default; -FieldSearchSpec& -FieldSearchSpec::operator=(FieldSearchSpec&& rhs) = default; +FieldSearchSpec::FieldSearchSpec(FieldSearchSpec&& rhs) noexcept = default; +FieldSearchSpec& FieldSearchSpec::operator=(FieldSearchSpec&& rhs) noexcept = default; FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & fname, VsmfieldsConfig::Fieldspec::Searchmethod searchDef, diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.h b/vsm/src/vespa/vsm/vsm/fieldsearchspec.h index f8631cc0ace..59fe737d480 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.h +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.h @@ -14,7 +14,8 @@ public: VsmfieldsConfig::Fieldspec::Searchmethod searchMethod, const vespalib::string & arg1, size_t maxLength); ~FieldSearchSpec(); - FieldSearchSpec& operator=(FieldSearchSpec&& rhs); + FieldSearchSpec(FieldSearchSpec&& rhs) noexcept; + FieldSearchSpec& operator=(FieldSearchSpec&& rhs) noexcept; const FieldSearcher & searcher() const { return *_searcher; } const vespalib::string & name() const { return _name; } FieldIdT id() const { return _id; } diff --git a/vtag.cmake b/vtag.cmake index eefbb638ab2..a7aaf433024 100644 --- a/vtag.cmake +++ b/vtag.cmake @@ -1,10 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -if(NOT EXISTS "${CMAKE_SOURCE_DIR}/dist/vtag.map") +if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map") message(FATAL_ERROR "dist/vtag.map does not exist, please run bootstrap.sh before configuring cmake" ) endif() function(get_vtag_define KEY) - file(STRINGS dist/vtag.map VALUE REGEX "${KEY}") + file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map VALUE REGEX "${KEY}") list(GET VALUE 0 LINE) separate_arguments(DATA UNIX_COMMAND "${LINE}") list(GET DATA 1 VALUE) diff --git a/zookeeper-command-line-client/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli index 5b23226dc5b..abfe9ff75cf 100755 --- a/zookeeper-command-line-client/src/main/sh/vespa-zkcli +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli @@ -83,7 +83,7 @@ usage() { echo "-nosudo do not use sudo when running command" } -sudo="sudo -u ${VESPA_USER}" +sudo="/usr/bin/sudo -u ${VESPA_USER}" while [ $# -gt 0 ]; do case $1 in -h|-help) usage; exit 0;; diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 2d8620f1028..6e8c82bc66e 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.4) add_subdirectory(zookeeper-server-3.5) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index fe182645681..edfbdbad02e 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,6 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.4</module> <module>zookeeper-server-3.5</module> </modules> <dependencies> diff --git a/zookeeper-server/zookeeper-server-3.4/pom.xml b/zookeeper-server/zookeeper-server-3.4/pom.xml deleted file mode 100644 index 756e2332c15..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/pom.xml +++ /dev/null @@ -1,74 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.4</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.4.14</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 5b4c0c11e80..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Writes zookeeper config and starts zookeeper server. - * - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { - - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); - private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; - static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; - private final Thread zkServerThread; - private final ZookeeperServerConfig zookeeperServerConfig; - - VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) { - this.zookeeperServerConfig = zookeeperServerConfig; - System.setProperty("zookeeper.jmx.log4j.disable", "true"); - System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); - System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); - - writeConfigToDisk(zookeeperServerConfig); - zkServerThread = new Thread(this, "zookeeper server"); - if (startServer) { - zkServerThread.start(); - } - } - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this(zookeeperServerConfig, true); - } - - private void writeConfigToDisk(ZookeeperServerConfig config) { - String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile()); - new File(configFilePath).getParentFile().mkdirs(); - try (FileWriter writer = new FileWriter(configFilePath)) { - writer.write(transformConfigToString(config)); - writeMyIdFile(config); - } catch (IOException e) { - throw new RuntimeException("Error writing zookeeper config", e); - } - } - - private String transformConfigToString(ZookeeperServerConfig config) { - StringBuilder sb = new StringBuilder(); - sb.append("tickTime=").append(config.tickTime()).append("\n"); - sb.append("initLimit=").append(config.initLimit()).append("\n"); - sb.append("syncLimit=").append(config.syncLimit()).append("\n"); - sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n"); - sb.append("snapCount=").append(config.snapshotCount()).append("\n"); - sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n"); - sb.append("clientPort=").append(config.clientPort()).append("\n"); - sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); - sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); - // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands - // Includes all available commands in 3.4, except 'wchc' and 'wchp' - // Mandatory when using ZooKeeper 3.5 - sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); - ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> addServerToCfg(sb, server)); - return sb.toString(); - } - - private void writeMyIdFile(ZookeeperServerConfig config) throws IOException { - if (config.server().size() > 1) { - try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) { - writer.write(config.myid() + "\n"); - } - } - } - - private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { - boolean found = false; - for (ZookeeperServerConfig.Server server : servers) { - if (myid == server.id()) { - found = true; - break; - } - } - if (!found) { - throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")"); - } - } - - private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) { - sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n"); - } - - private void shutdown() { - zkServerThread.interrupt(); - try { - zkServerThread.join(); - } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e); - } - } - - @Override - public void run() { - System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); - String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]); - log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); - org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); - } - - @Override - public void deconstruct() { - shutdown(); - super.deconstruct(); - } - - private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { - return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); - } - -} diff --git a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java deleted file mode 100644 index 1081c5fda61..00000000000 --- a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.io.IOUtils; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -/** - * Tests the zookeeper server. - */ -public class VespaZooKeeperServerImplTest { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void config_is_written_correctly_when_one_server() throws IOException { - File cfgFile = folder.newFile(); - File idFile = folder.newFile(); - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.myidFile(idFile.getAbsolutePath()); - builder.server(newServer(0, "foo", 123, 321)); - builder.myid(0); - createServer(builder); - validateConfigFileSingleHost(cfgFile); - validateIdFile(idFile, ""); - } - - @Test - public void config_is_written_correctly_when_multiple_servers() throws IOException { - File cfgFile = folder.newFile(); - File idFile = folder.newFile(); - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.server(newServer(0, "foo", 123, 321)); - builder.server(newServer(1, "bar", 234, 432)); - builder.server(newServer(2, "baz", 345, 543)); - builder.myidFile(idFile.getAbsolutePath()); - builder.myid(1); - createServer(builder); - validateConfigFileMultipleHosts(cfgFile); - validateIdFile(idFile, "1\n"); - } - - private void createServer(ZookeeperServerConfig.Builder builder) { - new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false); - } - - @Test(expected = RuntimeException.class) - public void require_that_this_id_must_be_present_amongst_servers() { - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.server(newServer(1, "bar", 234, 432)); - builder.server(newServer(2, "baz", 345, 543)); - builder.myid(0); - createServer(builder); - } - - @Test - public void juteMaxBufferCanBeSet() throws IOException { - ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); - builder.myid(0); - File idFile = folder.newFile(); - File cfgFile = folder.newFile(); - - builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost")); - builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); - builder.myidFile(idFile.getAbsolutePath()); - - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); - - final int max_buffer = 1; - builder.juteMaxBuffer(max_buffer); - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); - } - - private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) { - ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); - builder.id(id); - builder.hostname(hostName); - builder.electionPort(electionPort); - builder.quorumPort(quorumPort); - return builder; - } - - private void validateIdFile(File idFile, String expected) throws IOException { - String actual = IOUtils.readFile(idFile); - assertThat(actual, is(expected)); - } - - private void validateConfigFileSingleHost(File cfgFile) throws IOException { - String expected = - "tickTime=2000\n" + - "initLimit=20\n" + - "syncLimit=15\n" + - "maxClientCnxns=0\n" + - "snapCount=50000\n" + - "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + - "clientPort=2181\n" + - "autopurge.purgeInterval=1\n" + - "autopurge.snapRetainCount=15\n" + - "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + - "server.0=foo:321:123\n"; - validateConfigFile(cfgFile, expected); - } - - private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { - String expected = - "tickTime=2000\n" + - "initLimit=20\n" + - "syncLimit=15\n" + - "maxClientCnxns=0\n" + - "snapCount=50000\n" + - "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" + - "clientPort=2181\n" + - "autopurge.purgeInterval=1\n" + - "autopurge.snapRetainCount=15\n" + - "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + - "server.0=foo:321:123\n" + - "server.1=bar:432:234\n" + - "server.2=baz:543:345\n"; - validateConfigFile(cfgFile, expected); - } - - private void validateConfigFile(File cfgFile, String expected) throws IOException { - String actual = IOUtils.readFile(cfgFile); - assertThat(actual, is(expected)); - } -} diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java index 0981ae615df..863d2dba708 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java @@ -221,7 +221,7 @@ public class VespaZooKeeperServerImplTest { "ssl.quorum.clientAuth=NEED\n" + "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.quorum.enabledProtocols=TLSv1.2\n" + - "ssl.quorum.protocol=TLSv1.2\n"; + "ssl.quorum.protocol=TLS\n"; } private String commonTlsClientServerConfig() { @@ -229,7 +229,7 @@ public class VespaZooKeeperServerImplTest { "ssl.clientAuth=NEED\n" + "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.enabledProtocols=TLSv1.2\n" + - "ssl.protocol=TLSv1.2\n"; + "ssl.protocol=TLS\n"; } private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { |