diff options
557 files changed, 9161 insertions, 3545 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a4ef1bdba25..8aa853e8c39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,9 @@ vespa_use_default_cxx_compiler() vespa_use_default_java_home() project(vespa CXX C) +vespa_use_default_vespa_unprivileged() +vespa_use_default_cmake_install_prefix() +vespa_use_default_vespa_user() vespa_use_default_build_settings() # allows import of project in CLion on OSX diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index 5f9b1f51863..b200244a21c 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -2,6 +2,7 @@ package com.yahoo.application; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; @@ -117,11 +118,13 @@ public final class Application implements AutoCloseable { List<MlModelImporter> modelImporters = List.of(new VespaImporter(), new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); DeployState deployState = new DeployState.Builder() .applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), true)) .modelImporters(modelImporters) .deployLogger((level, s) -> { }) + .accessLoggingEnabledByDefault(false) .build(); return new VespaModel(new NullConfigModelRegistry(), deployState); } catch (IOException | SAXException e) { @@ -584,6 +587,7 @@ public final class Application implements AutoCloseable { xml.println("</search>"); } + xml.println("<accesslog type=\"disabled\" />"); xml.println("</container>"); } diff --git a/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/application/src/test/app-packages/searcher-app/services.xml b/application/src/test/app-packages/searcher-app/services.xml index 0fd37554c3b..5b9f4265b5a 100644 --- a/application/src/test/app-packages/searcher-app/services.xml +++ b/application/src/test/app-packages/searcher-app/services.xml @@ -12,4 +12,5 @@ </chain> </search> + <accesslog type="disabled" /> </container> diff --git a/application/src/test/app-packages/withcontent/services.xml b/application/src/test/app-packages/withcontent/services.xml index 251f591b6b8..4338579b49a 100644 --- a/application/src/test/app-packages/withcontent/services.xml +++ b/application/src/test/app-packages/withcontent/services.xml @@ -22,7 +22,7 @@ </chain> </document-processing> --> - + <accesslog type="disabled" /> </container> <content version="1.0" id="foo"> diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index f2af5537490..2c0b44c6d64 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -183,6 +183,7 @@ public class ApplicationTest { } @Test + // TODO: Creates access log public void renderer() throws Exception { try ( ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container() @@ -375,7 +376,8 @@ public class ApplicationTest { private static String servicesXmlWithServer(int port) { return "<container version='1.0'>" + " <http> <server port='" + port +"' id='foo'/> </http>" + - "</container>"; + " <accesslog type=\"disabled\" />" + + "</container>"; } @Test @@ -392,7 +394,8 @@ public class ApplicationTest { " <access-control domain='foo' />" + " </filtering>" + " </http>" + - "</container>"; + " <accesslog type=\"disabled\" />" + + "</container>"; } } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java index 529f6a60c0d..81f5a681f1a 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java @@ -44,6 +44,7 @@ public class ContainerDocprocTest { xml += " </chain>\n" + " </document-processing>\n" + + " <accesslog type=\"disabled\" />" + "</container>\n"; return xml; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java index 79510375414..3d7eed1e729 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java @@ -45,7 +45,7 @@ public class ContainerModelEvaluationTest { } private void assertLoadedModels(JDisc jdisc) { { - String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\"}"; + String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; assertResponse("http://localhost/model-evaluation/v1", expected, jdisc); } @@ -55,6 +55,11 @@ public class ContainerModelEvaluationTest { } { + String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval", expected, jdisc); + } + + { // Note: The specific response value here has not been verified String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.5066885003407351},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3912837743150205},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":-0.12401806321703948},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.7019029168606575},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.13120114146441697},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":0.6611923203384626},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.22365810810026446},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":-0.0740018307465809},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":0.056492490256153896},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.18422015072393733}]}"; assertResponse("http://localhost/model-evaluation/v1/tensorflow_mnist_saved/serving_default.y/eval?input=" + inputTensor(), expected, jdisc); diff --git a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java index 93ca09ac5fc..22c307c0a0c 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java @@ -31,6 +31,7 @@ public class ContainerProcessingTest { xml += " </chain>\n" + " </processing>\n" + + " <accesslog type=\"disabled\" />" + "</container>\n"; return xml; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java index 8f3e7693bc5..a66547e6ed9 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java @@ -33,6 +33,7 @@ public class ContainerRequestTest { binding + "</binding>\n" + " </handler>\n" + + " <accesslog type=\"disabled\" />" + "</container>"; } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java index d133b71b8da..793ee9ab094 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java @@ -8,6 +8,8 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import org.junit.Test; +import java.nio.charset.StandardCharsets; + import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -25,7 +27,7 @@ public class ContainerSearchTest { try (JDisc container = containerWithSearch(searcherId)) { byte[] rendered = container.search().processAndRender(ComponentSpecification.fromString("mychain"), ComponentSpecification.fromString("XmlRenderer"), new Query("")); - String renderedAsString = new String(rendered, "utf-8"); + String renderedAsString = new String(rendered, StandardCharsets.UTF_8); assertThat(renderedAsString, containsString(searcherId)); } } @@ -44,11 +46,12 @@ public class ContainerSearchTest { public JDisc containerWithSearch(String searcherId) { return JDisc.fromServicesXml("<container version=\"1.0\">" + // - "<search>" + // - "<chain id=\"mychain\">" + // - "<searcher id=\"" + searcherId + "\"/>" + // - "</chain>" + // - "</search>" + // + " <search>" + // + " <chain id=\"mychain\">" + // + " <searcher id=\"" + searcherId + "\"/>" + // + " </chain>" + // + " </search>" + // + " <accesslog type=\"disabled\" />" + // "</container>", Networking.disable); } diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java index c284087243d..63347c2475a 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java @@ -164,6 +164,7 @@ public class ContainerTest { "<http>\n" + // "<server id=\"main\" port=\"9999\" />\n" + // "</http>\n" + // + "<accesslog type=\"disabled\" />" + "</container>"; return JDisc.fromServicesXml(xml, Networking.disable); } diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java index 0eb308f124c..caddabff8ed 100644 --- a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java +++ b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java @@ -66,11 +66,10 @@ public class JerseyTest { @Test public void jersey_resources_in_provided_dependencies_can_be_invoked_from_application() throws Exception { - BundleClasspathMapping providedDependency = new BundleClasspathMapping(bundleSymbolicName, - Arrays.asList(testClassesDirectory)); + BundleClasspathMapping providedDependency = + new BundleClasspathMapping(bundleSymbolicName, List.of(testClassesDirectory)); - save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), - Arrays.asList(providedDependency))); + save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), List.of(providedDependency))); with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter)); } @@ -113,16 +112,16 @@ public class JerseyTest { } private interface ThrowingConsumer<T> { - public void accept(T arg) throws Exception; + void accept(T arg) throws Exception; } private interface HttpGetter { - public HttpResponse get(String path) throws Exception; + HttpResponse get(String path) throws Exception; } @SuppressWarnings("try") // jdisc unreferenced inside try private void with_jersey_resources(List<String> packagesToScan, ThrowingConsumer<HttpGetter> f) throws Exception { - StringBuffer packageElements = new StringBuffer(); + StringBuilder packageElements = new StringBuilder(); for (String p : packagesToScan) { packageElements.append("<package>"); packageElements.append(p); @@ -140,6 +139,7 @@ public class JerseyTest { "<http>" + // "<server id=\"mainServer\" port=\"0\" />" + // "</http>" + // + "<accesslog type=\"disabled\" />" + "</container>" + // "</services>", // Networking.enable)) { @@ -177,8 +177,8 @@ public class JerseyTest { } public void saveMainBundleClassPathMappings(String classPathElement) throws Exception { - BundleClasspathMapping mainBundleClassPathMappings = new BundleClasspathMapping(bundleSymbolicName, - Arrays.asList(classPathElement)); + BundleClasspathMapping mainBundleClassPathMappings = + new BundleClasspathMapping(bundleSymbolicName, List.of(classPathElement)); save(new ProjectBundleClassPaths(mainBundleClassPathMappings, emptyList())); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index a1984557c31..466a9783bb3 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -17,7 +17,7 @@ import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index d164555dc3e..0ac833b7aee 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java index c63ed3fceba..343a9feeed6 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java @@ -12,7 +12,7 @@ import com.yahoo.text.StringUtilities; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpUriRequest; diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index 0eda6bd946b..6783763b210 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -54,6 +54,7 @@ public class ContainerTester { private static String servicesXml() { return "<container version='1.0'>\n" + + " <accesslog type=\"disabled\"/>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>10</maxthreads>\n" + " </config>\n" + diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index f04e155aa2c..99d1b064a34 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 5e3e2a42ed5..27c7ed6c21b 100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -55,7 +55,7 @@ cmake3 \ -DJAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-openjdk} \ -DCMAKE_PREFIX_PATH="/opt/vespa-deps" \ -DEXTRA_LINK_DIRECTORY="/opt/vespa-deps/lib64;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ - -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION" \ + -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION;/usr/include/openblas" \ -DCMAKE_INSTALL_RPATH="${VESPA_INSTALL_PREFIX}/lib64;/opt/vespa-deps/lib64;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \ ${UNPRIVILEGED_ARGS} \ ${EXTRA_CMAKE_ARGS} \ diff --git a/build_settings.cmake b/build_settings.cmake index 679eed558d5..53fcbbad9a2 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # @author Vegard Sjonfjell -include(vtag.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/vtag.cmake) # Build options # Whether to build unit tests as part of the 'all' target diff --git a/client/pom.xml b/client/pom.xml index 3dee66f31ec..6f281fddf8c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -63,6 +63,21 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <finalName>${project.artifactId}-jar-with-dependencies</finalName> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 0f5a5e6271d..43527335802 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -879,7 +879,8 @@ "public java.util.Optional tlsSecrets()", "public java.util.Optional endpointCertificateSecrets()", "public abstract double defaultTermwiseLimit()", - "public abstract boolean useBucketSpaceMetric()" + "public abstract boolean useBucketSpaceMetric()", + "public boolean useNewAthenzFilter()" ], "fields": [] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 81ac02a5400..2410de55f86 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -59,6 +59,7 @@ public interface ModelContext { default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); + default boolean useNewAthenzFilter() { return false; } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 7c9e930bb4f..a9b4e06ae1b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -67,6 +67,7 @@ public class DeployState implements ConfigDefinitionStore { private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; private final Optional<Model> previousModel; + private final boolean accessLoggingEnabledByDefault; private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; @@ -101,14 +102,15 @@ public class DeployState implements ConfigDefinitionStore { Version vespaVersion, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, + Optional<Model> previousModel, Set<ContainerEndpoint> endpoints, Collection<MlModelImporter> modelImporters, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now, - Version wantedNodeVespaVersion) { + Version wantedNodeVespaVersion, + boolean accessLoggingEnabledByDefault) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -116,6 +118,7 @@ public class DeployState implements ConfigDefinitionStore { this.properties = properties; this.vespaVersion = vespaVersion; this.previousModel = previousModel; + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); this.documentModel = searchDocumentModel.getDocumentModel(); @@ -217,6 +220,10 @@ public class DeployState implements ConfigDefinitionStore { return logger; } + public boolean getAccessLoggingEnabledByDefault() { + return accessLoggingEnabledByDefault; + } + public FileRegistry getFileRegistry() { return fileRegistry; } @@ -289,6 +296,7 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private boolean accessLoggingEnabledByDefault = true; public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -360,6 +368,15 @@ public class DeployState implements ConfigDefinitionStore { return this; } + /** + * Whether access logging is enabled for an application without an accesslog element in services.xml. + * True by default. + */ + public Builder accessLoggingEnabledByDefault(boolean accessLoggingEnabledByDefault) { + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; + return this; + } + public DeployState build() { return build(new ValidationParameters()); } @@ -386,7 +403,8 @@ public class DeployState implements ConfigDefinitionStore { queryProfiles, semanticRules, now, - wantedNodeVespaVersion); + wantedNodeVespaVersion, + accessLoggingEnabledByDefault); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9f4d1b09f91..0b4562ecd5c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -4,8 +4,8 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useAdaptiveDispatch = false; private double defaultTermwiseLimit = 1.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); + private boolean useNewAthenzFilter = false; @Override public boolean multitenant() { return multitenant; } @@ -61,6 +62,7 @@ public class TestProperties implements ModelContext.Properties { @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } + @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; @@ -102,6 +104,16 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setUseNewAthenzFilter(boolean useNewAthenzFilter) { + this.useNewAthenzFilter = useNewAthenzFilter; + return this; + } + + public TestProperties setZone(Zone zone) { + this.zone = zone; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set<String> importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set<Name> documentReferences) { + public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set<Name> documentReferences) { + Set<Name> documentReferences, + Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 9ff749a994c..ade8ae21870 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -41,7 +41,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { } if (structUsedByField.getName().equals(structType.getName())) { //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0); + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..ee4fc41f2f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,33 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; + +import java.util.Collections; +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List<Search> searchDefinitions; + + public ImportedFieldsEnumerator(List<Search> searchDefinitions) { + this.searchDefinitions = searchDefinitions; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.searchDefinitions.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fa196cd3bbf..23eb814de81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -788,8 +788,7 @@ public class RankProfile implements Cloneable { type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + " with type " + field.getType().asTensorType() + - ", but this is already defined " + - "in another query profile with type " + + ", but this is already defined in another query profile with type " + context.getType(feature.get()))); context.setType(feature.get(), type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index 60b8ee78c7b..d8773063053 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -138,11 +138,10 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return Collections.singletonList(field); } if (fieldType instanceof ArrayDataType) { - boolean header = field.isHeader(); List<Field> ret = new LinkedList<>(); - Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header); + Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType()); for (Field flatField : flattenField(innerField)) { - ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header)); + ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()))); } return ret; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index c657d29033a..4fe6c3a96f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -38,9 +38,8 @@ import java.util.TreeMap; /** * The field class represents a document field. It is used in - * the Document class to get and set fields. Each SDField has - * a name, a numeric ID, a data type, and a boolean that says whether it's - * a header field. The numeric ID is used when the fields are stored + * the Document class to get and set fields. Each SDField has a name, a numeric ID, + * a data type. The numeric ID is used when the fields are stored * in serialized form. * * @author bratseth @@ -120,15 +119,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * Creates a new field. This method is only used to create reserved fields * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) { - super(name, id, dataType, isHeader); - populate(populate, repo, name, dataType, isHeader); + protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + super(name, id, dataType); + populate(populate, repo, name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) { - this(repo, name, id, dataType, isHeader, true); + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { + this(repo, name, id, dataType, true); } /** @@ -136,41 +134,35 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, @param name The name of the field @param dataType The datatype of the field - @param isHeader Whether this is a "header" field or a "content" field - (true = "header"). */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) { - super(name,dataType,isHeader); - populate(populate, repo, name, dataType, isHeader); + public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { + super(name,dataType); + populate(populate, repo, name, dataType); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) { - populate(populate,repo, name, dataType, isHeader, null, 0); + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { + populate(populate,repo, name, dataType, null, 0); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) { + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, isHeader, recursion); + populateWithStructFields(repo, name, dataType, recursion); populateWithStructMatching(repo, name, dataType, fieldMatching); } } - public SDField(String name, DataType dataType, boolean isHeader) { - this(null, name, dataType, isHeader, true); - } /** * Creates a new field. * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner the owning document (used to check for id collisions) */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; - populate(populate, repo, name, dataType, isHeader); + populate(populate, repo, name, dataType); } /** @@ -178,27 +170,25 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner The owning document (used to check for id collisions) * @param fieldMatching The matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion); + populate(populate, repo, name, dataType, fieldMatching, recursion); } /** - * Constructor for <b>header</b> fields * * @param name The name of the field * @param dataType The datatype of the field */ public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name,dataType,true, true); + this(repo, name,dataType, true); } public SDField(String name, DataType dataType) { this(null, name,dataType); @@ -277,7 +267,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) { + public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { DataType dt = getFirstStructOrMapRecursive(); if (dt == null) { return; @@ -286,11 +276,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, MapDataType mdt = (MapDataType) dataType; SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("key", keyField); SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("value", valueField); } else { if (recursion >= 10) { @@ -306,7 +296,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } for (Field field : subType.fieldSet()) { SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - isHeader, subType, new Matching(), true, recursion + 1); + subType, new Matching(), true, recursion + 1); structFields.put(field.getName(), subField); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 886bf777d3a..04d11792379 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ import com.yahoo.document.DataType; */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) { - super(owner, name, dataType, isHeader, owner, false); + public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { + super(owner, name, dataType, owner, false); } public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, true, false); + super(null, name, dataType, false); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index 6fdf448a39b..a6707ec7ac0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -27,6 +27,7 @@ public class ExpressionTransforms { ImmutableList.of(new TensorFlowFeatureConverter(), new OnnxFeatureConverter(), new XgboostFeatureConverter(), + new LightGBMFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), new FunctionInliner(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java new file mode 100644 index 00000000000..5bde627dc0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java @@ -0,0 +1,59 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the lightgbm(model-path) pseudofeature with the + * native Vespa ranking expression implementing the same computation. + * + * @author lesters + */ +public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("lightgbm")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedLightGBMModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + + "the LightGBM model file under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index d1eb18c4916..0ffd13927b4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -71,7 +71,7 @@ public class AddExtraFieldsToDocument extends Processor { if (docField == null) { ImmutableSDField existingField = search.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType(), true); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index 9a9e9bbba5f..89b8889b4ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.logging.Level; /** @@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor { public void process(boolean validate, boolean documentsOnly) { if (documentsOnly) return; + Set<Reference> warnedAbout = new HashSet<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { - resolveTypesIn(profile, validate); + resolveTypesIn(profile, validate, warnedAbout); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e); @@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor { * * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types */ - private void resolveTypesIn(RankProfile profile, boolean validate) { + private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) { MapEvaluationTypeContext context = profile.typeContext(queryProfiles); for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { ExpressionFunction expressionFunction = function.getValue().function(); @@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor { profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) { - deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " + + if ( context.tensorsAreUsed() && + ! context.queryFeaturesNotDeclared().isEmpty() && + ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { + deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() + + "' are not declared in query profile " + "types and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); + warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index d0a0bbfb748..d6398bc348c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ public class UriHack extends Processor { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType, true); + SDField partField = new SDField(partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java index 19a62f47e39..6950eb1fd84 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java @@ -84,6 +84,7 @@ public class DefaultPublicMetrics { metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 57d938f8a71..38f8cd67601 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -72,6 +72,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.server.network.tls-connections-broken")); metrics.add(new Metric("vds.server.network.failed-tls-config-reloads")); + // C++ Fnet metrics + metrics.add(new Metric("vds.server.fnet.num-connections")); + return metrics; } @@ -453,10 +456,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); @@ -468,10 +474,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java index 4f84a01ff94..4a331718985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.List; import java.util.Optional; import static com.yahoo.component.ComponentSpecification.fromString; @@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString; * Configure SSL using file references * * @author mortent + * @author bjorncs */ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer { public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; @@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C private final String certificatePath; private final String caCertificatePath; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + private final List<String> cipherSuites; + private final List<String> protocolVersions; - public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + public ConfiguredFilebasedSslProvider(String servername, + String privateKeyPath, + String certificatePath, + String caCertificatePath, + String clientAuthentication, + List<String> cipherSuites, + List<String> protocolVersions) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C this.certificatePath = certificatePath; this.caCertificatePath = caCertificatePath; this.clientAuthentication = mapToConfigEnum(clientAuthentication); + this.cipherSuites = cipherSuites; + this.protocolVersions = protocolVersions; } @Override public void getConfig(ConnectorConfig.Builder builder) { - builder.ssl.enabled(true); - builder.ssl.privateKeyFile(privateKeyPath); - builder.ssl.certificateFile(certificatePath); - builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); - builder.ssl.clientAuth(clientAuthentication); + builder.ssl( + new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyPath) + .certificateFile(certificatePath) + .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")) + .clientAuth(clientAuthentication) + .enabledCipherSuites(cipherSuites) + .enabledProtocols(protocolVersions)); } public SimpleComponent getComponent() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index db831a1ec2f..562026ab4dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider; +import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import org.w3c.dom.Element; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import static java.util.stream.Collectors.toList; + /** * @author Einar M R Rosenvinge * @author mortent @@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); + List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites"); + List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols"); return new ConfiguredFilebasedSslProvider( serverName, privateKeyFile, certificateFile, caCertificateFile.orElse(null), - clientAuthentication.orElse(null)); + clientAuthentication.orElse(null), + cipherSuites, + protocols); } else if (sslProviderConfigurator != null) { String className = sslProviderConfigurator.getAttribute("class"); String bundle = sslProviderConfigurator.getAttribute("bundle"); @@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil } } + private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) { + return XmlHelper.getOptionalChildValue(sslElement, listElementName) + .map(element -> + Arrays.stream(element.split(",")) + .filter(listEntry -> !listEntry.isBlank()) + .map(String::trim) + .collect(toList())) + .orElse(List.of()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index aef2697a5dd..9d7274f1bbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -302,7 +302,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } - if (accessLogElements.isEmpty() && cluster.getSearch() != null) + if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) cluster.addDefaultSearchAccessLog(); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 3c09a521715..eb7d7763cd1 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -637,7 +637,7 @@ void field(SDDocumentType document, Search search) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, document); + field = new TemporarySDField(name, type, document); } lbrace() (fieldBody(field, search, document) (<NL>)*)* <RBRACE> { @@ -931,7 +931,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, struct); + field = new TemporarySDField(name, type, struct); struct.addField(field); } lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index e3ad942e7b3..c47983adc12 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -37,7 +37,8 @@ OptionalDedicatedNodes = element nodes { attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & attribute dedicated { xsd:boolean }? & - attribute exclusive { xsd:boolean }? + attribute exclusive { xsd:boolean }? & + Resources? } GenericConfig = element config { diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 6e4346d96ee..726fa849c00 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -96,7 +96,9 @@ Ssl = element ssl { element private-key-file { string } & element certificate-file { string } & element ca-certificates-file { string }? & - element client-authentication { string "disabled" | string "want" | string "need" }? + element client-authentication { string "disabled" | string "want" | string "need" }? & + element cipher-suites { string }? & + element protocols { string }? } SslProvider = element ssl-provider { diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd index ab5e42f983d..247df8a0241 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd @@ -33,8 +33,12 @@ search test { expression: xgboost("xgboost_2_2") } + function my_lightgbm() { + expression: lightgbm("lightgbm_regression") + } + first-phase { - expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm } } diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7b50176625d --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -0,0 +1,104 @@ +enablecompression false +datatype[0].id 1381038251 +datatype[0].structtype[0].name "position" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].compresstype NONE +datatype[0].structtype[0].compresslevel 0 +datatype[0].structtype[0].compressthreshold 95 +datatype[0].structtype[0].compressminsize 800 +datatype[0].structtype[0].field[0].name "x" +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" +datatype[0].structtype[0].field[1].name "y" +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" +datatype[1].id 595216861 +datatype[1].referencetype[0].target_type_id -1318255918 +datatype[2].id 542332920 +datatype[2].referencetype[0].target_type_id 443162583 +datatype[3].id 959075962 +datatype[3].structtype[0].name "ad.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "campaign_ref" +datatype[3].structtype[0].field[0].datatype 595216861 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "person_ref" +datatype[3].structtype[0].field[1].datatype 542332920 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[4].id -255288561 +datatype[4].structtype[0].name "ad.body" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id 2987301 +datatype[5].documenttype[0].name "ad" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct 959075962 +datatype[5].documenttype[0].bodystruct -255288561 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].documenttype[0].importedfield[0].name "my_cool_field" +datatype[5].documenttype[0].importedfield[1].name "my_swag_field" +datatype[5].documenttype[0].importedfield[2].name "my_name" +datatype[6].id -2041471955 +datatype[6].structtype[0].name "campaign.header" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].compresstype NONE +datatype[6].structtype[0].compresslevel 0 +datatype[6].structtype[0].compressthreshold 95 +datatype[6].structtype[0].compressminsize 800 +datatype[6].structtype[0].field[0].name "cool_field" +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].detailedtype "" +datatype[6].structtype[0].field[1].name "swag_field" +datatype[6].structtype[0].field[1].datatype 4 +datatype[6].structtype[0].field[1].detailedtype "" +datatype[7].id 1448849794 +datatype[7].structtype[0].name "campaign.body" +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].compresstype NONE +datatype[7].structtype[0].compresslevel 0 +datatype[7].structtype[0].compressthreshold 95 +datatype[7].structtype[0].compressminsize 800 +datatype[8].id -1318255918 +datatype[8].documenttype[0].name "campaign" +datatype[8].documenttype[0].version 0 +datatype[8].documenttype[0].inherits[0].name "document" +datatype[8].documenttype[0].inherits[0].version 0 +datatype[8].documenttype[0].headerstruct -2041471955 +datatype[8].documenttype[0].bodystruct 1448849794 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[9].id 3129224 +datatype[9].structtype[0].name "person.header" +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].compresstype NONE +datatype[9].structtype[0].compresslevel 0 +datatype[9].structtype[0].compressthreshold 95 +datatype[9].structtype[0].compressminsize 800 +datatype[9].structtype[0].field[0].name "name" +datatype[9].structtype[0].field[0].datatype 2 +datatype[9].structtype[0].field[0].detailedtype "" +datatype[10].id -2003767395 +datatype[10].structtype[0].name "person.body" +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].compresstype NONE +datatype[10].structtype[0].compresslevel 0 +datatype[10].structtype[0].compressthreshold 95 +datatype[10].structtype[0].compressminsize 800 +datatype[11].id 443162583 +datatype[11].documenttype[0].name "person" +datatype[11].documenttype[0].version 0 +datatype[11].documenttype[0].inherits[0].name "document" +datatype[11].documenttype[0].inherits[0].version 0 +datatype[11].documenttype[0].headerstruct 3129224 +datatype[11].documenttype[0].bodystruct -2003767395 +datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7859703ffe0 --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -0,0 +1,141 @@ +enablecompression false +documenttype[0].id 2987301 +documenttype[0].name "ad" +documenttype[0].version 0 +documenttype[0].headerstruct 959075962 +documenttype[0].bodystruct -255288561 +documenttype[0].inherits[0].id 8 +documenttype[0].datatype[0].id 959075962 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ad.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 95 +documenttype[0].datatype[0].sstruct.compression.minsize 200 +documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" +documenttype[0].datatype[0].sstruct.field[0].id 23963250 +documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" +documenttype[0].datatype[0].sstruct.field[1].name "person_ref" +documenttype[0].datatype[0].sstruct.field[1].id 100779805 +documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" +documenttype[0].datatype[1].id -255288561 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "ad.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 95 +documenttype[0].datatype[1].sstruct.compression.minsize 200 +documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +documenttype[0].referencetype[0].id 595216861 +documenttype[0].referencetype[0].target_type_id -1318255918 +documenttype[0].referencetype[1].id 542332920 +documenttype[0].referencetype[1].target_type_id 443162583 +documenttype[0].importedfield[0].name "my_cool_field" +documenttype[0].importedfield[1].name "my_swag_field" +documenttype[0].importedfield[2].name "my_name" +documenttype[1].id -1318255918 +documenttype[1].name "campaign" +documenttype[1].version 0 +documenttype[1].headerstruct -2041471955 +documenttype[1].bodystruct 1448849794 +documenttype[1].inherits[0].id 8 +documenttype[1].datatype[0].id -2041471955 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "campaign.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 95 +documenttype[1].datatype[0].sstruct.compression.minsize 200 +documenttype[1].datatype[0].sstruct.field[0].name "cool_field" +documenttype[1].datatype[0].sstruct.field[0].id 1588702436 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" +documenttype[1].datatype[0].sstruct.field[1].name "swag_field" +documenttype[1].datatype[0].sstruct.field[1].id 1691224741 +documenttype[1].datatype[0].sstruct.field[1].datatype 4 +documenttype[1].datatype[0].sstruct.field[1].detailedtype "" +documenttype[1].datatype[1].id 1448849794 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "campaign.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 95 +documenttype[1].datatype[1].sstruct.compression.minsize 200 +documenttype[1].fieldsets{[document]}.fields[0] "cool_field" +documenttype[1].fieldsets{[document]}.fields[1] "swag_field" +documenttype[2].id 443162583 +documenttype[2].name "person" +documenttype[2].version 0 +documenttype[2].headerstruct 3129224 +documenttype[2].bodystruct -2003767395 +documenttype[2].inherits[0].id 8 +documenttype[2].datatype[0].id 3129224 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "person.header" +documenttype[2].datatype[0].sstruct.version 0 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 95 +documenttype[2].datatype[0].sstruct.compression.minsize 200 +documenttype[2].datatype[0].sstruct.field[0].name "name" +documenttype[2].datatype[0].sstruct.field[0].id 1160796772 +documenttype[2].datatype[0].sstruct.field[0].datatype 2 +documenttype[2].datatype[0].sstruct.field[0].detailedtype "" +documenttype[2].datatype[1].id -2003767395 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "person.body" +documenttype[2].datatype[1].sstruct.version 0 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 95 +documenttype[2].datatype[1].sstruct.compression.minsize 200 +documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/integration/lightgbm/models/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java new file mode 100644 index 00000000000..2c13427760f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 8378ec811a5..e46208c770d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to bar and add another field SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); @@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to non-existing document bar SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); fooDocument.addField(fooRefToBarField); @@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest { resolver.resolveReferences(fooDocument); } - private static void addAttributeAspect(SDField fooRefToBarField) { - fooRefToBarField.parseIndexingScript("{ attribute }"); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..fcbb89b5c42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + Search parentSearch = new Search(); + SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch); + var parentField = new SDField("their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSearch.addDocument(parentDocument); + + Search fooSearch = new Search(); + SDField fooRefToParent = new SDField( + "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + var fooImports = fooSearch.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); + fooSearch.addDocument(fooDocument); + + Search barSearch = new Search(); + SDField barRefToParent = new SDField( + "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + var barImports = barSearch.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType("bar", barSearch); + barSearch.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 08dd5148b29..0cd6674751e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import java.util.HashMap; @@ -33,6 +34,7 @@ class RankProfileSearchFixture { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); private final QueryProfileRegistry queryProfileRegistry; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 9e9ef2589be..a306e0f2c90 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true, logger); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..79d19371f1c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java new file mode 100644 index 00000000000..d0ee0523489 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; + +/** + * Utility functions for testing generated configs for reference/imported fields. + */ +public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase { + + private static String TEST_FOLDER = "src/test/configmodel/types/references/"; + + protected void assertDocumentConfigs(DocumentModel model, + String cfgFileSpec) throws IOException { + assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); + assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); + } + + protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + + protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java new file mode 100644 index 00000000000..599ae77a456 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase { + + @Test + public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException { + assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " field person_ref type reference<person> { indexing: attribute }", + " }", + " import field campaign_ref.cool_field as my_cool_field {}", + " import field campaign_ref.swag_field as my_swag_field {}", + " import field person_ref.name as my_name {}", + "}")), + "multiple_imported_fields"); + } + + private static class TestDocumentModelBuilder { + private final SearchBuilder builder = new SearchBuilder(); + public TestDocumentModelBuilder addCampaign() throws ParseException { + builder.importString(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); + return this; + } + public TestDocumentModelBuilder addPerson() throws ParseException { + builder.importString(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + return this; + } + public DocumentModel build(String adSdContent) throws ParseException { + builder.importString(adSdContent); + builder.build(); + return builder.getModel(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index e9a7a6ed33e..55980ee5fea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -1,15 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; import java.io.IOException; @@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals; /** * @author geirst */ -public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase { @Test public void reference_fields_can_reference_other_document_types() throws ParseException, IOException { @@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT assertEquals(campaignRefType.getTargetType(), campaignType); } - private static String TEST_FOLDER = "src/test/configmodel/types/references/"; - - private void assertDocumentConfigs(DocumentModel model, - String cfgFileSpec) throws IOException { - assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); - assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); - } - - private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { - DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); - assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); - } - - private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { - DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); - assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); - } - private static class TestDocumentModelBuilder { private final SearchBuilder builder = new SearchBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4064e53dfb7..3dfcef70aba 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest { headerfields, new StructDataType("bodyfields"), new FieldSets(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 4ea10c9a1c1..00ab175f496 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -18,30 +18,22 @@ import static org.junit.Assert.assertEquals; /** * @author gjoranv - * @since 5.5 */ public class AccessLogTest extends ContainerModelBuilderTestBase { @Test - public void default_access_log_is_only_added_when_search_is_present() { + public void default_access_log_is_added_by_default() { Element cluster1Elem = DomBuilderTest.parse( "<container id='cluster1' version='1.0'>", - "<search />", - nodesXml, - "</container>"); - Element cluster2Elem = DomBuilderTest.parse( - "<container id='cluster2' version='1.0'>", " <nodes>", " <node hostalias='mockhost' baseport='1234' />", " </nodes>", "</container>" ); - createModel(root, cluster1Elem, cluster2Elem); + createModel(root, cluster1Elem); assertNotNull(getJsonAccessLog("cluster1")); - assertNull( getJsonAccessLog("cluster2")); assertNull(getVespaAccessLog("cluster1")); - assertNull(getVespaAccessLog("cluster2")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 68f507c810d..0f9bd506310 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas " <client-authentication>need</client-authentication>", " </ssl>", " </server>", + " <server port='9003' id='with-ciphers-and-protocols'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>", + " <protocols>TLSv1.3</protocols>", + " </ssl>", + " </server>", " </http>", nodesXml, "", @@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo(""))); assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH))); + ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols"); + assertTrue(withCiphersAndProtocols.ssl().enabled()); + assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384")))); + assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3")))); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index ce36ecc4a1c..7a3b76db7f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; @@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -35,6 +36,7 @@ public class ImportedModelTester { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter(), new VespaImporter()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index c5c475360a3..ced7243adf5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -45,7 +45,7 @@ public class MlModelsTest { private void verify(VespaModel model) { assertEquals("Global models are created (although not used directly here", - 4, model.rankProfileList().getRankProfiles().size()); + 5, model.rankProfileList().getRankProfiles().size()); RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); model.getSearchClusters().get(0).getConfig(builder); @@ -71,8 +71,9 @@ public class MlModelsTest { "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + + "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" + "vespa.rank.firstphase: rankingExpression(firstphase)\n" + - "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" + + "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" + "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 3d4ac7f2eeb..2d3ddc33afb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -96,9 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); - assertEquals(4, config.rankprofile().size()); + assertEquals(5, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); + assertTrue(modelNames.contains("lightgbm_regression")); assertTrue(modelNames.contains("mnist_saved")); assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); @@ -112,13 +113,18 @@ public class ModelEvaluationTest { ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig)); - assertEquals(4, evaluator.models().size()); + assertEquals(5, evaluator.models().size()); Model xgboost = evaluator.models().get("xgboost_2_2"); assertNotNull(xgboost); assertNotNull(xgboost.evaluatorOf()); assertNotNull(xgboost.evaluatorOf("xgboost_2_2")); + Model lightgbm = evaluator.models().get("lightgbm_regression"); + assertNotNull(lightgbm); + assertNotNull(lightgbm.evaluatorOf()); + assertNotNull(lightgbm.evaluatorOf("lightgbm_regression")); + Model tensorflow_mnist = evaluator.models().get("mnist_saved"); assertNotNull(tensorflow_mnist); assertEquals(1, tensorflow_mnist.functions().size()); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 2bbd98f72ac..1bf42650123 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -119,6 +119,13 @@ <certificate-file>/foo/cert</certificate-file> <ca-certificates-file>/foo/cacerts</ca-certificates-file> <client-authentication>want</client-authentication> + <cipher-suites> + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + </cipher-suites> + <protocols>TLSv1.2,TLSv1.3</protocols> </ssl> </server> <server port="4083" id="sslProvider"> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 413e277655a..f66bacbc383 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -5,14 +5,13 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.ArrayList; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java new file mode 100644 index 00000000000..892ac639198 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -0,0 +1,17 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision.zone; + +/** + * The routing methods supported by a zone. + * + * @author mpolden + */ +public enum RoutingMethod { + + /** Routing happens through shared routing layer */ + shared, + + /** Routing happens through a dedicated layer 4 load balancer */ + exclusive, + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 46efe7a440d..5ef23c80eac 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; import com.yahoo.config.provision.CloudName; @@ -22,6 +22,9 @@ public interface ZoneFilter { /** Zones which support direct routing through exclusive load balancers. */ ZoneList directlyRouted(); + /** Zones where traffic is routed using given method */ + ZoneList routingMethod(RoutingMethod method); + /** Zones where config servers are up and running. */ ZoneList reachable(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java index 1069f1cdd53..7c042656bda 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -9,7 +9,6 @@ import com.yahoo.slime.Slime; * Implements a config instance serializer, serializing a config instance to a slime object. * * @author Ulf Lilleengen - * @since 5.1.14 */ public class ConfigInstanceSerializer implements Serializer { private final Slime slime; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java index fb2a3acbfdc..9710ee607eb 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -38,9 +38,7 @@ public class ConfigInstanceUtil { } } - public static <T extends ConfigInstance> T getNewInstance(Class<T> type, - String configId, - ConfigPayload payload) { + public static <T extends ConfigInstance> T getNewInstance(Class<T> type, String configId, ConfigPayload payload) { T instance; try { ConfigTransformer<?> transformer = new ConfigTransformer<>(type); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java index 1eaada5ae9d..121dba1e555 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -4,7 +4,6 @@ package com.yahoo.config.subscription; /** * This exception is thrown when any blocking call within the Config API is interrupted. * @author Ulf Lilleengen - * @since 5.1 */ @SuppressWarnings("serial") public class ConfigInterruptedException extends RuntimeException { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java index 57f4ad863f2..c3ceb81d3db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.config.ConfigKey; /** * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s * - * @author vegardh + * @author Vegard Havdal */ public class ConfigSet implements ConfigSource { private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java index d0eda9d27cc..0aac060abf6 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -4,7 +4,7 @@ package com.yahoo.config.subscription; /** * A type of source of config * - * @author vegardh + * @author Vegard Havdal */ public interface ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index ae6f14ad59e..5c0b932dcce 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -25,7 +25,7 @@ import static java.util.stream.Collectors.toList; * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. * - * @author vegardh + * @author Vegard Havdal */ public class ConfigSubscriber implements AutoCloseable { diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java index d90a0493838..6e12a38faa3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -5,8 +5,7 @@ import java.io.File; /** * Source specifying config from a local directory - * @author vegardh - * @since 5.1 + * @author Vegard Havdal * */ public class DirSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java index 27129853eae..9b6fb6442e8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -6,7 +6,7 @@ import java.io.File; /** * Source specifying config from one local file * - * @author vegardh + * @author Vegard Havdal */ public class FileSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java index 6275befe18c..a098dec7f21 100644 --- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -5,8 +5,7 @@ import java.util.jar.JarFile; /** * Source specifying config as a jar file entry - * @author vegardh - * @since 5.1 + * @author Vegard Havdal * */ public class JarSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java index d68b503eb74..099a6bdc334 100644 --- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -4,7 +4,7 @@ package com.yahoo.config.subscription; /** * Source specifying raw config, where payload is given programmatically * - * @author vegardh + * @author Vegard Havdal */ public class RawSource implements ConfigSource { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java index aa80cc75ef0..d3a1bfb50a9 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -11,8 +11,7 @@ import java.lang.reflect.Constructor; /** * Subscription on a programmatically built set of configs - * @author vegardh - * @since 5.1 + * @author Vegard Havdal */ public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index c02301a0c17..3bf6093e872 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.config.protocol.DefContent; /** * Represents one active subscription to one config * - * @author vegardh + * @author Vegard Havdal */ public abstract class ConfigSubscription<T extends ConfigInstance> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index bcee06cd667..60c920d1dfa 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -18,7 +18,7 @@ import com.yahoo.log.LogLevel; /** * Subscription used when config id is file:... * - * @author vegardh + * @author Vegard Havdal */ public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java index bf247dffb7c..a40da85c88b 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.config.RawConfig; /** * A config handle which does not use the config class, but payload instead. Used in config proxy. * - * @author vegardh + * @author Vegard Havdal */ @SuppressWarnings({"rawtypes", "unchecked"}) public class GenericConfigHandle extends ConfigHandle { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java index 82d4708f53b..324546230d9 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.config.TimingValues; /** * A subscriber that can subscribe without the class. Used by configproxy. * - * @author vegardh + * @author Vegard Havdal */ public class GenericConfigSubscriber extends ConfigSubscriber { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index c1f9ce02650..0887ed9aad5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. * - * @author vegardh + * @author Vegard Havdal * */ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfig> { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 06c772ac456..e9daafb779c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -233,7 +233,8 @@ public class JRTConfigRequester implements RequestWaiter { suspendWarningLogged = Instant.MIN; noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); - sub.setLastCallBackOKTS(System.currentTimeMillis()); + sub.setLastCallBackOKTS(Instant.now()); + log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index d9366c28b9b..39e6c69f539 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -28,8 +30,8 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester requester; private TimingValues timingValues; - // Last time we got an OK JRT callback for this - private long lastOK = 0; + // Last time we got an OK JRT callback + private Instant lastOK = Instant.MIN; /** * The queue containing either nothing or the one (newest) request that has got callback from JRT, @@ -40,9 +42,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { super(key, subscriber); - this.timingValues=timingValues; + this.timingValues = timingValues; if (source instanceof ConfigSourceSet) { - this.sources=(ConfigSourceSet) source; + this.sources = (ConfigSourceSet) source; } } @@ -53,7 +55,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc ConfigState<T> configState = getConfigState(); boolean gotNew = configState.isGenerationChanged() || configState.isConfigChanged() || hasException(); // Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check - if (getReqQueue().peek()==null && gotNew) { + if (getReqQueue().peek() == null && gotNew) { return true; } // Otherwise poll the queue for another generation or timeout @@ -88,6 +90,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // timed out, we know nothing new. return false; } + log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -135,11 +138,11 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc @Override public boolean subscribe(long timeout) { - lastOK=System.currentTimeMillis(); + lastOK = Instant.now(); requester = getRequester(); requester.request(this); JRTClientConfigRequest req = reqQueue.peek(); - while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) { + while (req == null && (Instant.now().isBefore(lastOK.plus(Duration.ofMillis(timeout))))) { try { Thread.sleep(10); } catch (InterruptedException e) { @@ -152,7 +155,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc private JRTConfigRequester getRequester() { JRTConfigRequester requester = subscriber.requesters().get(sources); - if (requester==null) { + if (requester == null) { requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); subscriber.requesters().put(sources, requester); } @@ -164,8 +167,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc public void close() { super.close(); reqQueue = new LinkedBlockingQueue<>() { + @SuppressWarnings("NullableProblems") @Override - public void put(JRTClientConfigRequest e) throws InterruptedException { + public void put(JRTClientConfigRequest e) { // When closed, throw away all requests that callbacks try to put } }; @@ -191,7 +195,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); } - void setLastCallBackOKTS(long lastCallBackOKTS) { + void setLastCallBackOKTS(Instant lastCallBackOKTS) { this.lastOK = lastCallBackOKTS; } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index a96499482c5..05da9a72837 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.config.ConfigPayload; /** * Subscription to use when config id is jar:.../foo.jar[!/pathInJar/] * - * @author vegardh + * @author Vegard Havdal * @author gjoranv */ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { @@ -33,8 +33,8 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // jar:configs/app.jar!/configs/ JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) { super(key, subscriber); - this.jarName=jarName; - this.path=path; + this.jarName = jarName; + this.path = path; } @Override @@ -43,17 +43,18 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // Not supporting changing the payload for jar return true; } - if (zipEntry==null) { + if (zipEntry == null) { // First time polled - JarFile jarFile = null; + JarFile jarFile; try { jarFile = new JarFile(jarName); } catch (IOException e) { throw new IllegalArgumentException(e); } zipEntry = getEntry(jarFile, path); - if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); - T config = null; + if (zipEntry == null) + throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); + T config; try { ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n"))); config = payload.toInstance(configClass, key.getConfigId()); @@ -78,6 +79,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } return false; } + /** * Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in * the given JarFile. diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java index 9ad8f5c6ba2..58eed7f9e78 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.util.ConfigUtils; * For unit testing * * @author hmusum - * @since 5.1.11 */ public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index bc546cfdd1f..68ff6bb0135 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -12,10 +12,10 @@ import com.yahoo.vespa.config.ConfigPayload; /** * Subscription used when config id is raw:... - * + * <p> * Config is the actual text given after the config id, with newlines * - * @author vegardh + * @author Vegard Havdal */ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { @@ -24,7 +24,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) { super(key, subscriber); - this.inputPayload=pl; + this.inputPayload = pl; } @Override @@ -32,7 +32,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc if (checkReloaded()) { return true; } - if (payload==null) { + if (payload == null) { payload = inputPayload; ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId())); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java index 55cb05ec230..6fcdc19cf97 100755 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -78,7 +78,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable if (!(o instanceof ConfigKey)) { return false; } - ConfigKey<?> key = (ConfigKey) o; + ConfigKey<?> key = (ConfigKey<?>) o; return (name.equals(key.name) && configId.equals(key.configId) && namespace.equals(key.namespace)); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java index 4f02df04b9b..f4858843574 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java @@ -20,7 +20,6 @@ import java.io.OutputStream; * A class that holds a representation of a config payload. * * @author Ulf Lilleengen - * @since 5.1.6 */ public class ConfigPayload { private final Slime slime; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 65106f158fc..97e85f1a39b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -33,7 +33,6 @@ import java.util.logging.Logger; * TODO: This can be refactored a lot, since many of the reflection methods are duplicated * * @author Ulf Lilleengen, hmusum, Tony Vaagenes - * @since 5.1.6 */ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName()); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java index bb974ddae42..2e470ec55e4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java @@ -12,7 +12,6 @@ import java.util.*; * * TODO: Add toString * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigPayloadBuilder { diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index 163e010cdd6..6d2ae3ef13e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -12,7 +12,6 @@ import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer; * A utility class that can be used to transform config from one format to another. * * @author Ulf Lilleengen, hmusum, Tony Vaagenes - * @since 5.1.6 */ public class ConfigTransformer<T extends ConfigInstance> { /** diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java index f34b66e17a8..5bbc985cce0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java @@ -11,7 +11,6 @@ import com.yahoo.slime.*; * TODO: Support giving correct type of default values * * @author Ulf Lilleengen - * @since 5.1 */ public class DefaultValueApplier { @@ -31,21 +30,11 @@ public class DefaultValueApplier { } private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) { - cursor.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - applyDefaultsToObject(cursor.field(name), def); - } - }); + cursor.traverse((ObjectTraverser) (name, inspector) -> applyDefaultsToObject(cursor.field(name), def)); } private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) { - cursor.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - applyDefaultsToObject(cursor.entry(idx), def); - } - }); + cursor.traverse((ArrayTraverser) (idx, inspector) -> applyDefaultsToObject(cursor.entry(idx), def)); } private void applyDefaultsToObject(Cursor cursor, InnerCNode def) { diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java index c151121f0e9..3da09d07016 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java +++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java @@ -5,7 +5,6 @@ package com.yahoo.vespa.config; * Interface for counters. * * @author Ulf Lilleengen - * @since 5.9 */ public interface GenerationCounter { /** diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java index 2fbbda2aa38..2f351cc2bd4 100644 --- a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigInstance; /** * - /** * A generic config with an internal generic builder that mimics a real config builder in order to support builders * when we don't have the schema. * diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java index 7ac03ea6151..05125f8b08b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java @@ -11,7 +11,6 @@ import javax.tools.JavaFileObject; * TODO: Assumes that diagnostics is the same as given to the task, not ideal. * * @author Ulf Lilleengen - * @since 5.2 */ class CompilationTask { private final JavaCompiler.CompilationTask task; diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java index 80909ab653d..fa5cf427aad 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance; * Represents a builder that can be instantiated. * * @author Ulf Lilleengen - * @since 5.2 */ public interface CompiledBuilder { <BUILDER extends ConfigInstance.Builder> BUILDER newInstance(); diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java index 1939a007059..c67d2121844 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java @@ -8,7 +8,6 @@ import java.net.URI; * Represents an in memory source object that can be compiled. * * @author Ulf Lilleengen - * @since 5.2 */ class StringSourceObject extends SimpleJavaFileObject { private final String code; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java index ba9fa601c81..bc6f97a3e00 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java @@ -10,7 +10,6 @@ import java.io.IOException; * Contains info relevant for compression and decompression. * * @author Ulf Lilleengen - * @since 5.19 */ public class CompressionInfo { private static final String COMPRESSION_TYPE = "compressionType"; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java index c6548f63cd7..5fbd61cde3d 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.protocol; /** * @author Ulf Lilleengen - * @since 5.18 */ public enum CompressionType { UNCOMPRESSED, LZ4; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java index c07be8337fe..bb0ee2bb935 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -5,7 +5,6 @@ import com.yahoo.text.Utf8Array; import java.io.IOException; import java.io.OutputStream; -import java.util.List; /** * A config response encapsulates the payload and some meta information. This makes it possible to @@ -19,8 +18,6 @@ public interface ConfigResponse { Utf8Array getPayload(); - List<String> getLegacyPayload(); - long getGeneration(); boolean isInternalRedeploy(); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java index ea2cf1a1bd8..3aa1898fcd1 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java @@ -12,7 +12,6 @@ import java.util.List; /** * @author Ulf Lilleengen -* @since 5.3 */ public class DefContent { private final List<String> data; @@ -35,12 +34,7 @@ public class DefContent { static DefContent fromSlime(Inspector data) { final List<String> lst = new ArrayList<>(); - data.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - lst.add(inspector.asString()); - } - }); + data.traverse((ArrayTraverser) (idx, inspector) -> lst.add(inspector.asString())); return new DefContent(lst); } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java index 25fecba425c..ab47fec0641 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -64,14 +64,6 @@ public interface JRTClientConfigRequest extends JRTConfigRequest { String getNewConfigMd5(); /** - * Test whether or not the payload is contained in this response or not. - * Should return false for error responses as well. - * - * @return true if empty, false if not. - */ - boolean containsPayload(); - - /** * Test whether or not the response contains an updated config or not. * False if no response has been returned. * diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java index 1c842a4d1b0..b3ca7355023 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -5,13 +5,20 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigSubscription; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.util.ConfigUtils; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.Optional; +import java.util.logging.Logger; /** * Represents version 3 config request for config clients. Provides methods for inspecting request and response @@ -21,7 +28,12 @@ import java.util.Optional; * * @author Ulf Lilleengen */ -public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { +public class JRTClientConfigRequestV3 implements JRTClientConfigRequest { + + protected static final Logger log = Logger.getLogger(JRTClientConfigRequestV3.class.getName()); + protected final SlimeRequestData requestData; + protected final Request request; + private final SlimeResponseData responseData; protected JRTClientConfigRequestV3(ConfigKey<?> key, String hostname, @@ -32,15 +44,38 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { - super(key, hostname, defSchema, configMd5, generation, timeout, trace, compressionType, vespaVersion); + Slime data = SlimeRequestData.encodeRequest(key, + hostname, + defSchema, + configMd5, + generation, + timeout, + trace, + getProtocolVersion(), + compressionType, + vespaVersion); + Request jrtReq = new Request(getJRTMethodName()); + jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data))); + + this.requestData = new SlimeRequestData(jrtReq, data); + this.responseData = new SlimeResponseData(jrtReq); + this.request = jrtReq; + } + + protected static String encodeAsUtf8String(Slime data) { + ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + try { + new JsonFormat(true /* compact format */).encode(baos, data); + } catch (IOException e) { + throw new RuntimeException("Unable to encode config request", e); + } + return Utf8.toString(baos.toByteArray()); } - @Override protected String getJRTMethodName() { return JRTMethods.configV3getConfigMethodName; } - @Override protected boolean checkReturnTypes(Request request) { return JRTMethods.checkV3ReturnTypes(request); } @@ -133,4 +168,142 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { return requestData.getVespaVersion(); } + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()) + .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) + .append("'\n"); + sb.append("response='").append(getNewConfigMd5()) + .append(",").append(getNewGeneration()) + .append(",").append(responseIsInternalRedeploy()) + .append("'\n"); + return sb.toString(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + @Override + public boolean hasUpdatedGeneration() { + long prevGen = getRequestGeneration(); + long newGen = getNewGeneration(); + return ConfigUtils.isGenerationNewer(newGen, prevGen); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + protected String newConfMd5() { + String newMd5 = getNewConfigMd5(); + if ("".equals(newMd5)) return getRequestConfigMd5(); + return newMd5; + } + + protected long newGen() { + long newGen = getNewGeneration(); + if (newGen == 0) return getRequestGeneration(); + return newGen; + } + + @Override + public DefContent getDefContent() { + return requestData.getSchema(); + } + + @Override + public boolean isError() { + return request.isError(); + } + + @Override + public boolean hasUpdatedConfig() { + String respMd5 = getNewConfigMd5(); + return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); + } + + @Override + public Trace getResponseTrace() { + return responseData.getResponseTrace(); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + @Override + public boolean validateResponse() { + if (request.isError()) { + return false; + } else if (request.returnValues().size() == 0) { + return false; + } else if (!checkReturnTypes(request)) { + log.warning("Invalid return types for config response: " + errorMessage()); + return false; + } + return true; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + return (errorCode == 0); + } + + @Override + public String getNewConfigMd5() { + return responseData.getResponseConfigMd5(); + } + + @Override + public long getNewGeneration() { + return responseData.getResponseConfigGeneration(); + } + + @Override + public boolean responseIsInternalRedeploy() { + return responseData.getResponseInternalRedeployment(); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + protected SlimeResponseData getResponseData() { + return responseData; + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java index 9dec28b13e0..0861c5008c0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java @@ -12,7 +12,6 @@ import java.util.*; * To hide JRT implementations. * * @author Ulf Lilleengen - * @since 5.3 */ public class JRTConfigRequestFactory { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java index f70ebf39a28..83befda7a59 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -1,17 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Value; import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; /** - * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields JRT + * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields: * * * A metadata field containing json data describing config generation, md5 and compression info * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload @@ -22,14 +30,41 @@ import java.io.IOException; * * @author Ulf Lilleengen */ -// TODO: Merge with parent -public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { +public class JRTServerConfigRequestV3 implements JRTServerConfigRequest { + protected static final Logger log = Logger.getLogger(JRTServerConfigRequestV3.class.getName()); + private static final JsonFactory jsonFactory = new JsonFactory(); + protected final Request request; + private final SlimeRequestData requestData; /** Response field */ private boolean internalRedeploy = false; + // Response values + private boolean isDelayed = false; + private Trace requestTrace = null; protected JRTServerConfigRequestV3(Request request) { - super(request); + this.requestData = new SlimeRequestData(request); + this.request = request; + } + + protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { + return jsonFactory.createGenerator(byteArrayOutputStream); + } + + protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { + return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { + jsonGenerator.writeStringField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { + jsonGenerator.writeNumberField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException { + jsonGenerator.writeBooleanField(fieldName, value); } @Override @@ -85,4 +120,145 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { return new JRTServerConfigRequestV3(req); } + @Override + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public DefContent getDefContent() { + return getSchema(); + } + + @Override + public boolean noCache() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()).append("'\n"); + return sb.toString(); + } + + @Override + public Payload payloadFromResponse(ConfigResponse response) { + return Payload.from(response.getPayload(), response.getCompressionInfo()); + } + + private DefContent getSchema() { + return requestData.getSchema(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + public Trace getRequestTrace() { + if (requestTrace == null) { + requestTrace = requestData.getRequestTrace(); + } + return requestTrace; + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + if (errorCode != 0) { + addErrorResponse(errorCode); + } + return (errorCode == 0); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + private void addErrorResponse(int errorCode) { + addErrorResponse(errorCode, ErrorCode.getName(errorCode)); + } + + @Override + public void setDelayedResponse(boolean delayedResponse) { + this.isDelayed = delayedResponse; + } + + @Override + public void addErrorResponse(int errorCode, String name) { + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); + try { + JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); + jsonWriter.writeStartObject(); + addCommonReturnValues(jsonWriter); + jsonWriter.writeEndObject(); + jsonWriter.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add error response for " + this); + } + request.setError(errorCode, name); + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + } + + protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { + ConfigKey<?> key = requestData.getConfigKey(); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); + jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); + jsonGenerator.writeRawValue(getRequestTrace().toString(true)); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + @Override + public boolean isDelayedResponse() { + return isDelayed; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + protected CompressionType getCompressionType() { + return requestData.getCompressionType(); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java index 79746127bcd..61a742ade3b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java @@ -7,7 +7,6 @@ import java.io.ByteArrayOutputStream; * Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method. * * @author Ulf Lilleengen - * @since 5.19 */ class NoCopyByteArrayOutputStream extends ByteArrayOutputStream { public NoCopyByteArrayOutputStream() { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index fed9a96504d..8c7c7f2703e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -14,7 +14,6 @@ import java.util.regex.Pattern; * Static utility methods for verifying common request properties. * * @author Ulf Lilleengen - * @since 5.3 */ public class RequestValidation { private static final Logger log = Logger.getLogger(RequestValidation.class.getName()); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java deleted file mode 100644 index 3ccf71ec519..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.protocol; - -import com.yahoo.jrt.Request; -import com.yahoo.jrt.StringValue; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.util.ConfigUtils; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of - * payload encoding and decoding, as well as adding extra request/response fields. - * - * @author Ulf Lilleengen - */ -public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest { - - protected static final Logger log = Logger.getLogger(SlimeClientConfigRequest.class.getName()); - - protected final SlimeRequestData requestData; - private final SlimeResponseData responseData; - - protected final Request request; - - protected SlimeClientConfigRequest(ConfigKey<?> key, - String hostname, - DefContent defSchema, - String configMd5, - long generation, - long timeout, - Trace trace, - CompressionType compressionType, - Optional<VespaVersion> vespaVersion) { - Slime data = SlimeRequestData.encodeRequest(key, - hostname, - defSchema, - configMd5, - generation, - timeout, - trace, - getProtocolVersion(), - compressionType, - vespaVersion); - Request jrtReq = new Request(getJRTMethodName()); - jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true))); - - this.requestData = new SlimeRequestData(jrtReq, data); - this.responseData = new SlimeResponseData(jrtReq); - this.request = jrtReq; - } - - protected abstract String getJRTMethodName(); - - protected static String encodeAsUtf8String(Slime data, boolean compact) { - ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); - try { - new JsonFormat(compact).encode(baos, data); - } catch (IOException e) { - throw new RuntimeException("Unable to encode config request", e); - } - return Utf8.toString(baos.toByteArray()); - } - - public ConfigKey<?> getConfigKey() { - return requestData.getConfigKey(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("request='").append(getConfigKey()) - .append(",").append(getClientHostName()) - .append(",").append(getRequestConfigMd5()) - .append(",").append(getRequestGeneration()) - .append(",").append(getTimeout()) - .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse("")) - .append("'\n"); - sb.append("response='").append(getNewConfigMd5()) - .append(",").append(getNewGeneration()) - .append(",").append(responseIsInternalRedeploy()) - .append("'\n"); - return sb.toString(); - } - - @Override - public String getClientHostName() { - return requestData.getClientHostName(); - } - - @Override - public Request getRequest() { - return request; - } - - @Override - public int errorCode() { - return request.errorCode(); - } - - @Override - public String errorMessage() { - return request.errorMessage(); - } - - @Override - public String getShortDescription() { - return toString(); - } - - @Override - public boolean hasUpdatedGeneration() { - long prevGen = getRequestGeneration(); - long newGen = getNewGeneration(); - return ConfigUtils.isGenerationNewer(newGen, prevGen); - } - - @Override - public long getTimeout() { - return requestData.getTimeout(); - } - - protected String newConfMd5() { - String newMd5 = getNewConfigMd5(); - if ("".equals(newMd5)) return getRequestConfigMd5(); - return newMd5; - } - - protected long newGen() { - long newGen = getNewGeneration(); - if (newGen==0) return getRequestGeneration(); - return newGen; - } - - @Override - public DefContent getDefContent() { - return requestData.getSchema(); - } - - @Override - public boolean isError() { - return request.isError(); - } - - @Override - public boolean containsPayload() { - return false; - } - - @Override - public boolean hasUpdatedConfig() { - String respMd5 = getNewConfigMd5(); - return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); - } - - @Override - public Trace getResponseTrace() { - return responseData.getResponseTrace(); - } - - @Override - public String getRequestConfigMd5() { - return requestData.getRequestConfigMd5(); - } - - @Override - public boolean validateResponse() { - if (request.isError()) { - return false; - } else if (request.returnValues().size() == 0) { - return false; - } else if (!checkReturnTypes(request)) { - log.warning("Invalid return types for config response: " + errorMessage()); - return false; - } - return true; - } - - @Override - public boolean validateParameters() { - int errorCode = RequestValidation.validateRequest(this); - return (errorCode == 0); - } - - protected abstract boolean checkReturnTypes(Request request); - - @Override - public String getNewConfigMd5() { - return responseData.getResponseConfigMd5(); - } - - @Override - public long getNewGeneration() { - return responseData.getResponseConfigGeneration(); - } - - @Override - public boolean responseIsInternalRedeploy() { - return responseData.getResponseInternalRedeployment(); - } - - @Override - public long getRequestGeneration() { - return requestData.getRequestGeneration(); - } - - protected SlimeResponseData getResponseData() { - return responseData; - } - - public Optional<VespaVersion> getVespaVersion() { - return requestData.getVespaVersion(); - } - -} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java index 327acab53d3..ff0b7f964bf 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -1,17 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.text.Utf8Array; -import com.yahoo.vespa.config.ConfigFileFormat; import com.yahoo.vespa.config.ConfigPayload; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; /** * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. @@ -22,27 +16,24 @@ public class SlimeConfigResponse implements ConfigResponse { private final Utf8Array payload; private final CompressionInfo compressionInfo; - private final InnerCNode targetDef; private final long generation; private final boolean internalRedeploy; private final String configMd5; - public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation, boolean internalRedeploy, String configMd5) { Utf8Array data = payload.toUtf8Array(true); - return new SlimeConfigResponse(data, targetDef, generation, internalRedeploy, + return new SlimeConfigResponse(data, generation, internalRedeploy, configMd5, CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); } public SlimeConfigResponse(Utf8Array payload, - InnerCNode targetDef, long generation, boolean internalRedeploy, String configMd5, CompressionInfo compressionInfo) { this.payload = payload; - this.targetDef = targetDef; this.generation = generation; this.internalRedeploy = internalRedeploy; this.configMd5 = configMd5; @@ -55,19 +46,6 @@ public class SlimeConfigResponse implements ConfigResponse { } @Override - public List<String> getLegacyPayload() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ConfigFileFormat format = new ConfigFileFormat(targetDef); - Payload v1payload = Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); - try { - ConfigPayload.fromUtf8Array(v1payload.getData()).serialize(baos, format); - return Arrays.asList(baos.toString(StandardCharsets.UTF_8).split("\\n")); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override public long getGeneration() { return generation; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java deleted file mode 100644 index 34d6f90cbcb..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.protocol; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.yahoo.jrt.*; -import com.yahoo.slime.*; -import com.yahoo.text.Utf8Array; -import com.yahoo.vespa.config.*; -import com.yahoo.vespa.config.ErrorCode; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Optional; -import java.util.logging.Logger; - -/** - * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of - * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3 - * config protocol. - * - * @author Ulf Lilleengen - */ -abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { - - protected static final Logger log = Logger.getLogger(SlimeServerConfigRequest.class.getName()); - - private static final JsonFactory jsonFactory = new JsonFactory(); - - private final SlimeRequestData requestData; - - // Response values - private boolean isDelayed = false; - private Trace requestTrace = null; - protected final Request request; - - protected SlimeServerConfigRequest(Request request) { - this.requestData = new SlimeRequestData(request); - this.request = request; - } - - protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { - return jsonFactory.createGenerator(byteArrayOutputStream); - } - - @Override - public ConfigKey<?> getConfigKey() { - return requestData.getConfigKey(); - } - - @Override - public DefContent getDefContent() { - return getSchema(); - } - - @Override - public boolean noCache() { - return false; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("request='").append(getConfigKey()) - .append(",").append(getClientHostName()) - .append(",").append(getRequestConfigMd5()) - .append(",").append(getRequestGeneration()) - .append(",").append(getTimeout()).append("'\n"); - return sb.toString(); - } - - @Override - public Payload payloadFromResponse(ConfigResponse response) { - return Payload.from(response.getPayload(), response.getCompressionInfo()); - } - - private DefContent getSchema() { - return requestData.getSchema(); - } - - @Override - public String getClientHostName() { - return requestData.getClientHostName(); - } - - public Trace getRequestTrace() { - if (requestTrace == null) { - requestTrace = requestData.getRequestTrace(); - } - return requestTrace; - } - - @Override - public Request getRequest() { - return request; - } - - @Override - public boolean validateParameters() { - int errorCode = RequestValidation.validateRequest(this); - if (errorCode != 0) { - addErrorResponse(errorCode); - } - return (errorCode == 0); - } - - @Override - public String getRequestConfigMd5() { - return requestData.getRequestConfigMd5(); - } - - private void addErrorResponse(int errorCode) { - addErrorResponse(errorCode, ErrorCode.getName(errorCode)); - } - - @Override - public void setDelayedResponse(boolean delayedResponse) { - this.isDelayed = delayedResponse; - } - - @Override - public void addErrorResponse(int errorCode, String name) { - ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); - try { - JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); - jsonWriter.writeStartObject(); - addCommonReturnValues(jsonWriter); - jsonWriter.writeEndObject(); - jsonWriter.close(); - } catch (IOException e) { - throw new IllegalArgumentException("Could not add error response for " + this); - } - request.setError(errorCode, name); - request.returnValues().add(createResponseValue(byteArrayOutputStream)); - } - - protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { - return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); - } - - protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { - ConfigKey<?> key = requestData.getConfigKey(); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); - setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); - jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); - jsonGenerator.writeRawValue(getRequestTrace().toString(true)); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { - jsonGenerator.writeStringField(fieldName, value); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { - jsonGenerator.writeNumberField(fieldName, value); - } - - protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException { - jsonGenerator.writeBooleanField(fieldName, value); - } - - @Override - public long getRequestGeneration() { - return requestData.getRequestGeneration(); - } - - @Override - public boolean isDelayedResponse() { - return isDelayed; - } - - @Override - public int errorCode() { - return request.errorCode(); - } - - @Override - public String errorMessage() { - return request.errorMessage(); - } - - @Override - public String getShortDescription() { - return toString(); - } - - protected CompressionType getCompressionType() { - return requestData.getCompressionType(); - } - - @Override - public long getTimeout() { - return requestData.getTimeout(); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return requestData.getVespaVersion(); - } - -} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java index f4754f4b0e1..998d2c8bcf5 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java @@ -9,7 +9,6 @@ import com.yahoo.yolean.trace.TraceNode; * Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode} * * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceDeserializer { private final Inspector entry; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java index 62fa6f8a882..5f5f057a26b 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -12,7 +12,6 @@ import java.util.Stack; * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. * * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceSerializer extends TraceVisitor { static final String TIMESTAMP = "timestamp"; @@ -30,7 +29,6 @@ public class SlimeTraceSerializer extends TraceVisitor { current.setLong(TIMESTAMP, node.timestamp()); encodePayload(current, node.payload()); addChildrenCursors(current, node); - } private void encodePayload(Cursor current, Object payload) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java index 17740c9f40d..fbeafdc3f6f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java @@ -13,7 +13,6 @@ import java.time.Clock; * A trace utility that can serialize/deserialize to/from {@link Slime} * * @author Ulf Lilleengen - * @since 5.3 */ public class Trace { private static final String TRACE_TRACELOG = "traceLog"; diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index d7653572773..8f856ff4771 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -33,6 +33,7 @@ import java.util.regex.Pattern; * Utilities for mangling config text, finding md5sums, finding name and namespace in .def files etc. */ public class ConfigUtils { + /* Patterns used for finding ranges in config definitions */ private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); @@ -88,6 +89,7 @@ public class ConfigUtils { /** * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * * @param str String to strip spaces from * @return String with spaces stripped */ @@ -103,15 +105,15 @@ public class ConfigUtils { } if (!inSpaceSequence) { // start of space sequence - inSpaceSequence=true; + inSpaceSequence = true; ret.append(" "); } } else { if (inSpaceSequence) { - inSpaceSequence=false; + inSpaceSequence = false; } - if (c=='\"') { - inQuotes=!inQuotes; + if (c == '\"') { + inQuotes = !inQuotes; } ret.append(c); } @@ -121,7 +123,7 @@ public class ConfigUtils { /** * Computes Md5 hash of a list of strings with the contents of a def-file. - * + * <p> * Each string is normalized according to the * rules of Vespa config definition files before they are used: * <ol> @@ -132,12 +134,12 @@ public class ConfigUtils { * <li>Remove 'version=<version-number>'</li> * </ol> * - * @param lines A list of lines constituting a def-file + * @param lines A list of lines constituting a def-file * @return the Md5 hash of the list, with lowercase letters */ public static String getDefMd5(List<String> lines) { List<String> linesCopy = new ArrayList<>(lines); - for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) { + for (Iterator<String> it = linesCopy.iterator(); it.hasNext(); ) { String line = it.next().trim(); if (! line.startsWith("#") && ! line.equals("")) { if (line.startsWith("version")) { @@ -169,7 +171,7 @@ public class ConfigUtils { } if (line.length() > 0) { line = stripSpaces(line); - m = spaceBeforeCommaPatter.matcher(line); + m = spaceBeforeCommaPatter.matcher(line); line = m.replaceAll(","); // Remove space before comma (for enums) sb.append(line).append("\n"); } @@ -188,7 +190,7 @@ public class ConfigUtils { public static String getDefNamespace(Reader in) { List<String> defLines = getDefLines(in); String defPackage = getDefKeyword(defLines, "package"); - if (! defPackage.isEmpty()) return defPackage; + if (!defPackage.isEmpty()) return defPackage; return getDefKeyword(defLines, "namespace"); } @@ -285,7 +287,7 @@ public class ConfigUtils { */ public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException { String[] fileName = file.getName().split("\\."); - assert(fileName.length >= 2); + assert (fileName.length >= 2); String name = fileName[fileName.length - 2]; byte[] content = IOUtils.readFileBytes(file); @@ -295,7 +297,7 @@ public class ConfigUtils { /** * Creates a ConfigDefinitionKey from a name and the content of a config definition * - * @param name the name of the config definition + * @param name the name of the config definition * @param content content of a config definition * @return a ConfigDefinitionKey */ @@ -306,6 +308,7 @@ public class ConfigUtils { /** * Escapes a config value according to the cfg format. + * * @param input the string to escape * @return the escaped string */ @@ -350,10 +353,10 @@ public class ConfigUtils { * Loop through values and return the first one that is set and non-empty. * * @param defaultValue The default value to use if no environment variables are set. - * @param envVars one or more environment variable strings + * @param envVars one or more environment variable strings * @return a String with the value of the environment variable */ - public static String getEnvValue(String defaultValue, String ... envVars) { + public static String getEnvValue(String defaultValue, String... envVars) { String value = null; for (String envVar : envVars) { if (value == null || value.isEmpty()) { diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java index d4280c6300f..c8693227920 100644 --- a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; /** * @author hmusum * @author Vegard Sjonfjell - * @since 5.1 */ public class CfgConfigPayloadBuilderTest { @@ -135,7 +134,7 @@ public class CfgConfigPayloadBuilderTest { " 'boolVal': 'true'", " },", " {", - " 'stringVal': 'blue a=\\\'escaped\\\'',", + " 'stringVal': 'blue a=\\'escaped\\'',", " 'boolVal': 'false'", " }", " ],", @@ -182,7 +181,7 @@ public class CfgConfigPayloadBuilderTest { assertDeserializedConfigEqualsJson("a b=\"escaped\"", inputJson( "{", - " 'a': 'b=\\\'escaped\\\''", + " 'a': 'b=\\'escaped\\''", "}" ) ); @@ -305,7 +304,7 @@ public class CfgConfigPayloadBuilderTest { } private static void assertDeserializedConfigEqualsJson(String serializedConfig, String expectedJson) { - assertDeserializedConfigEqualsJson(Arrays.asList(serializedConfig), expectedJson); + assertDeserializedConfigEqualsJson(List.of(serializedConfig), expectedJson); } private static void assertDeserializedConfigEqualsJson(List<String> inputConfig, String expectedJson) { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java index 2d0a40bcdc4..49fd25b6476 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; /** * @author gjoranv - * @since 5.1.6 */ public class ConfigInstancePayloadTest { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java index 8d9b2d96b04..95ffb31fe7d 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java @@ -3,7 +3,6 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; import com.yahoo.foo.FunctionTestConfig; -import com.yahoo.config.codegen.DefLine; import com.yahoo.vespa.config.ConfigPayload; import org.junit.Test; @@ -18,14 +17,6 @@ import static org.junit.Assert.assertThat; */ public class ConfigInstanceSerializationTest { - private DefLine.Type stringType = new DefLine.Type("string"); - private DefLine.Type intType = new DefLine.Type("int"); - private DefLine.Type longType = new DefLine.Type("long"); - private DefLine.Type boolType = new DefLine.Type("bool"); - private DefLine.Type doubleType = new DefLine.Type("double"); - private DefLine.Type fileType = new DefLine.Type("file"); - private DefLine.Type refType = new DefLine.Type("reference"); - @Test public void require_symmetrical_serialization_and_deserialization_with_builder() { FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); @@ -40,218 +31,4 @@ public class ConfigInstanceSerializationTest { assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2))); } -/** Looks like everything in the commented block tests unused api's.. remove? - - @Test - public void testSerializeAgainstConfigDefinitionAllowNothing() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - List<String> payload = Validator.serialize(config, def); - Assert.assertEquals(payload.size(), 0); - } - - @Test - public void testSerializeAgainstConfigDefinitionAllLeaves() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("bool_val", LeafCNode.newInstance(boolType, def, "bool_val")); - def.children().put("bool_with_def", LeafCNode.newInstance(boolType, def, "bool_with_def")); - def.children().put("int_val", LeafCNode.newInstance(intType, def, "int_val")); - def.children().put("int_with_def", LeafCNode.newInstance(intType, def, "int_with_def")); - def.children().put("long_val", LeafCNode.newInstance(longType, def, "long_val")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("double_val", LeafCNode.newInstance(doubleType, def, "double_val")); - def.children().put("double_with_def", LeafCNode.newInstance(doubleType, def, "double_with_def")); - def.children().put("string_val", LeafCNode.newInstance(stringType, def, "string_val")); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("enum_val", LeafCNode.newInstance(enumType(new String[]{"FOO", "BAR", "FOOBAR"}), def, "enum_val")); - def.children().put("enumwithdef", LeafCNode.newInstance(enumType(new String[]{"FOO2", "BAR2", "FOOBAR2"}), def, "enumwithdef", "BAR2")); - def.children().put("refval", LeafCNode.newInstance(refType, def, "refval")); - def.children().put("refwithdef", LeafCNode.newInstance(refType, def, "refwithdef")); - def.children().put("fileVal", LeafCNode.newInstance(fileType, def, "fileVal")); - - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*bool_val false.*")); - Assert.assertTrue(plString.matches(".*bool_with_def true.*")); - Assert.assertTrue(plString.matches(".*int_val 5.*")); - Assert.assertTrue(plString.matches(".*int_with_def -14.*")); - Assert.assertTrue(plString.matches(".*long_val 12345678901.*")); - Assert.assertTrue(plString.matches(".*long_with_def -9876543210.*")); - Assert.assertTrue(plString.matches(".*double_val 41\\.23.*")); - Assert.assertTrue(plString.matches(".*double_with_def -12.*")); - Assert.assertTrue(plString.matches(".*string_val \"foo\".*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertTrue(plString.matches(".*enum_val FOOBAR.*")); - Assert.assertTrue(plString.matches(".*enumwithdef BAR2.*")); - Assert.assertTrue(plString.matches(".*refval \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*refwithdef \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*fileVal \"etc\".*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionSomeLeaves() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - // But not double_with_def etc, and no structs/arrays - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - } - - @Test - public void testSerializationAgainstConfigDefinitionAddedValsInDef() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("someotherstring", LeafCNode.newInstance(stringType, def, "someotherstring", "some other")); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("some_other_long", LeafCNode.newInstance(longType, def, "some_other_long", "88")); - def.children().put("some_other_enum", LeafCNode.newInstance(enumType(new String[]{"hey", "ho", "lets", "go"}), def, "some_other_enum", "lets")); - def.children().put("int_val_nofdef", LeafCNode.newInstance(intType, def, "int_val_nodef", null)); - - // But not double_with_def etc, and no structs/arrays - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertTrue(plString.matches(".*.someotherstring \"some other\".*")); - Assert.assertTrue(plString.matches(".*some_other_long 88.*")); - Assert.assertTrue(plString.matches(".*some_other_enum lets.*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - Assert.assertFalse(plString.matches(".*int_val_nodef.*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionMismatchAllWays() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - - // Create all sorts of mismatches in the def schema used to serialize - InnerCNode def = new InnerCNode("function-test"); - def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); - def.children().put("stringwithdef", LeafCNode.newInstance(intType, def, "stringwithdef")); - def.children().put("basicStruct", LeafCNode.newInstance(intType, def, "basicStruct")); - InnerCNode doubleValWrong = new InnerCNode("double_val"); - doubleValWrong.children().put("foo", LeafCNode.newInstance(intType, def, "foo")); - def.children().put("double_val", doubleValWrong); - InnerCNode myArray = new InnerCNode("myarray"); - myArray.children().put("intval", LeafCNode.newInstance(stringType, myArray, "foo")); - InnerCNode myStruct = new InnerCNode("myStruct"); - myStruct.children().put("a", LeafCNode.newInstance(stringType, myStruct, "foo")); - myArray.children().put("myStruct", myStruct); - def.children().put("myarray", myArray); - - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertTrue(plString.matches(".*long_with_def.*")); - Assert.assertFalse(plString.matches(".*stringwithdef.*")); - Assert.assertFalse(plString.matches(".*basicStruct.*")); - Assert.assertFalse(plString.matches(".*double_val.*")); - Assert.assertFalse(plString.matches(".*intval.*")); - Assert.assertFalse(plString.matches(".*\\.a.*")); - } - - @Test - public void testSerializeAgainstConfigDefinitionComplex() { - FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); - - // Build a pretty complex def programatically - InnerCNode def = new InnerCNode("function-test"); - def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); - def.children().put("someUnknownStringNoDefault", LeafCNode.newInstance(stringType, def, "someUnknownStringNoDefault")); - InnerCNode basicStruct = new InnerCNode("basicStruct"); - basicStruct.children().put("foo", LeafCNode.newInstance(stringType, def, "foo")); // but not bar - InnerCNode rootStruct = new InnerCNode("rootStruct"); - InnerCNode inner1 = new InnerCNode("inner1"); - InnerCNode someUnknwonStruct = new InnerCNode("someUnknownStruct"); - InnerCNode someUnknownInner = new InnerCNode("someUnknownInner"); - InnerCNode innerArr = new InnerCNode("innerArr"); - rootStruct.children().put("inner1", inner1); - rootStruct.children().put("someUnknownStruct", someUnknwonStruct); - rootStruct.children().put("someUnknownInner", someUnknownInner); - rootStruct.children().put("innerArr", innerArr); - InnerCNode myarray = new InnerCNode("myarray"); - InnerCNode unknownInner = new InnerCNode("unknownInner"); - def.children().put("basicStruct", basicStruct); - def.children().put("rootStruct", rootStruct); - def.children().put("myarray", myarray); - def.children().put("unknownInner", unknownInner); - inner1.children().put("index", LeafCNode.newInstance(intType, inner1, "index")); - inner1.children().put("someUnknownInt", LeafCNode.newInstance(intType, inner1, "someUnknownInt", "-98")); - inner1.children().put("someUnknownIntNoDefault", LeafCNode.newInstance(intType, inner1, "someUnknownIntNoDefault")); - inner1.children().put("someUnknownEnum", LeafCNode.newInstance(enumType(new String[]{"goo", "go", "gorilla"}), inner1, "someUnknownEnum", "go")); - inner1.children().put("someUnknownEnumNoDefault", LeafCNode.newInstance(enumType(new String[]{"foo", "bar", "baz"}), inner1, "someUnknownEnumNoDefault")); - someUnknwonStruct.children().put("anint", LeafCNode.newInstance(intType, someUnknwonStruct, "anint", "3"));// But no instances of this in config - someUnknownInner.children().put("along", LeafCNode.newInstance(longType, someUnknownInner, "along", "234"));// No instance in config - innerArr.children().put("boolVal", LeafCNode.newInstance(boolType, innerArr, "boolVal")); - innerArr.children().put("someUnknownDouble", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDouble", "-675.789")); - innerArr.children().put("someUnknownDoubleNoDefault", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDoubleNoDefault")); - myarray.children().put("fileVal", LeafCNode.newInstance(fileType, myarray, "fileVal")); - myarray.children().put("stringval", new InnerCNode("stringval[]")); - // TODO make sure default for file is not allowed - //myarray.children().put("someUnknownFile", LeafCNode.newInstance(fileType, myarray, "someUnknownFile", "opt/")); - unknownInner.children().put("aDouble", LeafCNode.newInstance(doubleType, unknownInner, "aDouble", "1234")); - def.children().put("longarr", new InnerCNode("longarr[]")); - def.children().put("boolarr", new InnerCNode("boolarr[]")); - def.children().put("doublearr", new InnerCNode("doublearr[]")); - def.children().put("stringarr", new InnerCNode("stringarr[]")); - def.children().put("fileArr", new InnerCNode("fileArr[]")); - def.children().put("refarr", new InnerCNode("refarr[]")); - def.children().put("enumarr", new InnerCNode("enumarr[]")); - List<String> payload = Validator.serialize(config, def); - String plString = payload.toString(); - Assert.assertFalse(plString.matches(".*long_with_def \\-9876543210.*")); - Assert.assertFalse(plString.matches(".*someUnknownStringNoDefault.*")); - Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); - Assert.assertFalse(plString.matches(".*double_with_def.*")); - Assert.assertFalse(plString.matches(".*fileVal etc.*")); - Assert.assertTrue(plString.matches(".*basicStruct\\.foo \"basicFoo\".*")); - Assert.assertFalse(plString.matches(".*basicStruct\\.bar.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.inner0.*")); - Assert.assertFalse(plString.matches(".*unknownInner.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownStruct.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownInner.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.inner1\\.name.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.index 12.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownInt -98.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownEnum go.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.boolVal true.*")); - Assert.assertFalse(plString.matches(".*someUnknownEnumNoDefault.*")); - Assert.assertFalse(plString.matches(".*someUnknownDoubleNoDefault.*")); - Assert.assertFalse(plString.matches(".*someUnknownIntNoDefault.*")); - Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.someUnknownDouble -675.789.*")); - Assert.assertFalse(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.stringVal*")); - Assert.assertFalse(plString.matches(".*myarray\\[0\\].intval.*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].fileVal \"file0\".*")); - //assertTrue(plString.matches(".*myarray\\[0\\].someUnknownFile \"opt/\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[0\\] \"baah\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[1\\] \"yikes\".*")); - Assert.assertTrue(plString.matches(".*myarray\\[1\\].fileVal \"file1\".*")); - Assert.assertFalse(plString.matches(".*myarray\\[1\\].enumVal.*")); - Assert.assertFalse(plString.matches(".*myarray\\[1\\].refVal.*")); - Assert.assertTrue(plString.matches(".*boolarr\\[0\\] false.*")); - Assert.assertTrue(plString.matches(".*longarr\\[0\\] 9223372036854775807.*")); - Assert.assertTrue(plString.matches(".*longarr\\[1\\] -9223372036854775808.*")); - Assert.assertTrue(plString.matches(".*doublearr\\[0\\] 2344\\.0.*")); - Assert.assertTrue(plString.matches(".*doublearr\\[1\\] 123\\.0.*")); - Assert.assertTrue(plString.matches(".*stringarr\\[0\\] \"bar\".*")); - Assert.assertTrue(plString.matches(".*enumarr\\[0\\] VALUES.*")); - Assert.assertTrue(plString.matches(".*refarr\\[0\\] \\:parent\\:.*")); - Assert.assertTrue(plString.matches(".*refarr\\[1\\] \\:parent.*")); - Assert.assertTrue(plString.matches(".*refarr\\[2\\] parent\\:.*")); - Assert.assertTrue(plString.matches(".*fileArr\\[0\\] \"bin\".*")); - } - - private DefLine.Type enumType(String[] strings) { - return new DefLine.Type("enum").setEnumArray(strings); - } -**/ } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java index ee8682efe3c..7c08d0175a1 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java @@ -21,9 +21,9 @@ import static org.junit.Assert.fail; /** * @author Ulf Lilleengen * @author Vegard Sjonfjell - * @since 5.1 */ public class ConfigInstanceSerializerTest { + @Test public void test_that_leaf_types_are_serialized_to_json_types() { SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); @@ -225,4 +225,5 @@ public class ConfigInstanceSerializerTest { assertJsonEquals(baos.toString(), expectedJson); } + } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java index 1da53e4c3b9..4da0c3f51e0 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java @@ -12,7 +12,7 @@ import static org.junit.Assert.fail; /** * Tests different aspects of the ConfigInstance class and its underlying Nodes. * - * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + * @author gjoranv */ public class ConfigInstanceTest { private ConfigSourceSet sourceSet = new ConfigSourceSet("config-instance-test"); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java index 078be819d33..3bdaee09eaf 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java @@ -22,7 +22,6 @@ import static com.yahoo.foo.FunctionTestConfig.*; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigInstanceUtilTest { diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java index 35f4f14b4f4..a80da63afc9 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java @@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigInterruptedExceptionTest { + @Test public void require_that_throwable_is_preserved() { ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo")); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java index a120f19c17b..48a51bbc02c 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java @@ -10,9 +10,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigSetTest { + @Test public void testToString() { ConfigSet set = new ConfigSet(); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java index 38d4a6a4571..0879c330f45 100755 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.*; * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> */ public class ConfigSourceSetTest { + @Test public void testEquals() { assertEquals(new ConfigSourceSet(), new ConfigSourceSet()); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java index 43e3cf658f3..555434837c8 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java @@ -12,9 +12,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigURITest { + @Test public void testDefaultUri() { ConfigURI uri = ConfigURI.createFromId("foo"); diff --git a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java index 85346264837..1bcf09a4028 100644 --- a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; * @author hmusum */ public class DefaultConfigTest { + static final String CONFIG_ID = "raw:" + "nondefaultstring ####-------missing--------\n" + "defaultstring \"thedefault\"\n" + diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index cb25de89bfb..08d215670db 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.*; * Test cases for the "generic" (class-less) subscription mechanism. * * @author Ulf Lilleengen - * @since 5.1 */ public class GenericConfigSubscriberTest { @@ -52,7 +51,6 @@ public class GenericConfigSubscriberTest { GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78"); assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79"); - } @Test(expected=UnsupportedOperationException.class) diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java index 5e60c273ee4..430de894629 100644 --- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -11,6 +11,7 @@ import static org.junit.Assert.assertThat; * @author gjoranv */ public class NamespaceTest { + @Test public void verifyConfigClassWithExplicitNamespace() { NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java index 6512b5ea29e..f1bd164c874 100644 --- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertEquals; * @author Harald Musum */ public class UnicodeTest { + /** * Reads a config from a file which is exactly like one returned from * the config server given only default values for this config. diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java index 2f550fc8e1e..71e7a9f08d7 100644 --- a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1.7 */ public class FileConfigSubscriptionTest { private File TEST_TYPES_FILE; diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java index c77985c91d8..137d2894164 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertThat; * SEO keywords: test override() on builders. overrideTest, testOverride * * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigBuilderMergeTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java index bb65fdaa153..fce61cc802c 100755 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.*; * @author hmusum */ public class ConfigCacheKeyTest { + @Test public void testConfigCacheKey() { final String defMd5 = "md5"; diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java index dba73223097..810b9f58829 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java @@ -12,7 +12,6 @@ import java.io.IOException; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; - /** * Unit tests for ConfigDefinitionBuilder. * diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java index 3cc030d944b..5198759d3e2 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java @@ -24,7 +24,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigFileFormatterTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java index eb0c47e3b0a..c13d3ec9b9b 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ConfigPayloadBuilderTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java index f1b0adc03e7..b3db6e2ab43 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java @@ -24,8 +24,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author Ulf Lilleengen 3 - * @since 5.1 + * @author Ulf Lilleengen */ public class ConfigPayloadTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java index e67489b4030..6ab3059f1c0 100644 --- a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java @@ -16,9 +16,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class DefaultValueApplierTest { + public Slime apply(Slime slime, String ... extraFields) { StringBuilder defBuilder = new StringBuilder(); defBuilder.append("namespace=test").append("\n"); diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java index 8e6cc1fc6a0..64210eaa4b8 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java @@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat; /** * @author hmusum - * @since 5.1.9 */ public class ErrorCodeTest { + @Test public void basic() { assertThat(ErrorCode.getName(ErrorCode.INTERNAL_ERROR), is("INTERNAL_ERROR")); diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java index 450d7e8d6d8..a0b2c1185a7 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java @@ -8,7 +8,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ErrorTypeTest { @@ -32,4 +31,5 @@ public class ErrorTypeTest { assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL)); assertThat(ErrorType.getErrorType(0xdeadc0de), is(ErrorType.FATAL)); } + } diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java index ee7845086c2..2c60ff95fee 100644 --- a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java @@ -13,9 +13,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class GenericConfigBuilderTest { + @Test public void require_that_builder_can_be_overridden() throws IOException { ConfigPayloadBuilder ba = new ConfigPayloadBuilder(); diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java index a3125c73bea..805efc90ade 100644 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java @@ -15,7 +15,6 @@ import java.nio.file.Files; * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge. * * @author Ulf Lilleengen - * @since 5.12 */ public class LZ4CompressionTest { private static LZ4Factory factory = LZ4Factory.safeInstance(); diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java index 06bbaad4271..635d6a96a7c 100644 --- a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java @@ -9,9 +9,9 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.19 */ public class LZ4PayloadCompressorTest { + @Test public void testCompression() { assertCompression("hei hallo der"); diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index a564fea8b2e..3f6c54ea46e 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -113,11 +113,11 @@ public class RawConfigTest { assertThat(config.getDefMd5(), is(defMd5)); config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty()); assertNull(config.getDefMd5()); - config = new RawConfig(key, null, payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, null, payload, configMd5, generation, false,List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty()); assertThat(config.getDefMd5(), is("")); - config = new RawConfig(key, "", payload, configMd5, generation, false, Arrays.asList(""), Optional.empty()); + config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty()); assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); } diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java index 3b97c360c4b..0be5e98ff0a 100644 --- a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertThat; * @author hmusum */ public class TimingValuesTest { + @Test public void basic() { TimingValues tv = new TimingValues(); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java index 91adc544d88..a56c7ef2daa 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.config.protocol; import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.text.StringUtilities; import com.yahoo.text.Utf8Array; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.LZ4PayloadCompressor; @@ -12,10 +9,10 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.StringReader; -import java.util.List; +import java.nio.charset.StandardCharsets; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -27,17 +24,16 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_is_initialized() throws IOException { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); - ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, false, "mymd5"); - List<String> payload = response.getLegacyPayload(); + ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, 3, false, "mymd5"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + String payload = baos.toString(StandardCharsets.UTF_8); assertNotNull(payload); - assertThat(payload.size(), is(6)); - assertThat(payload.get(0), is("boolval false")); + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload); assertThat(response.getGeneration(), is(3L)); assertThat(response.getConfigMd5(), is("mymd5")); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -45,19 +41,15 @@ public class ConfigResponseTest { @Test public void require_that_slime_response_decompresses_on_serialize() throws IOException { ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); Utf8Array data = configPayload.toUtf8Array(true); Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.getBytes())); - ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); - List<String> payload = response.getLegacyPayload(); + ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + String payload = baos.toString(StandardCharsets.UTF_8); assertNotNull(payload); - assertThat(payload.size(), is(6)); - assertThat(payload.get(0), is("boolval false")); - assertThat(response.getGeneration(), is(3L)); - assertThat(response.getConfigMd5(), is("mymd5")); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos = new ByteArrayOutputStream(); response.serialize(baos, CompressionType.UNCOMPRESSED); assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java index 75ba1392fd1..5fcbd76b822 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.Before; import org.junit.Test; -import java.io.IOException; import java.util.Collections; import java.util.Optional; @@ -36,7 +35,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.3 */ public abstract class JRTConfigRequestBase { @@ -54,19 +52,19 @@ public abstract class JRTConfigRequestBase { protected JRTServerConfigRequest serverReq; @Before - public void setupRequest() throws IOException { + public void setupRequest() { clientReq = createReq(); serverReq = createReq(clientReq.getRequest()); assertTrue(serverReq.validateParameters()); } - private JRTClientConfigRequest createReq() throws IOException { + private JRTClientConfigRequest createReq() { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace); } - private JRTClientConfigRequest createReq(Payload payload) throws IOException { + private JRTClientConfigRequest createReq(Payload payload) { trace = Trace.createNew(3, new ManualClock()); trace.trace(1, "hei"); return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); @@ -195,7 +193,7 @@ public abstract class JRTConfigRequestBase { } @Test - public void payload_is_empty() throws IOException { + public void payload_is_empty() { Payload payload = Payload.from(ConfigPayload.empty()); clientReq = createReq(payload); serverReq = createReq(clientReq.getRequest()); @@ -209,7 +207,6 @@ public abstract class JRTConfigRequestBase { @Test public void request_interface_is_implemented() { JRTClientConfigRequest request = clientReq; - assertFalse(request.containsPayload()); assertFalse(request.isError()); assertThat(request.errorCode(), is(clientReq.getRequest().errorCode())); assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage())); @@ -258,7 +255,7 @@ public abstract class JRTConfigRequestBase { protected abstract String getProtocolVersion(); @Test - public void created_from_raw() throws IOException { + public void created_from_raw() { RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5); long serverTimeout = 100000L; JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9)); @@ -269,9 +266,8 @@ public abstract class JRTConfigRequestBase { assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent())); } - @Test - public void parameters_are_validated() throws IOException { + public void parameters_are_validated() { assertTrue(serverReq.validateParameters()); assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java index d50a92efc1a..6d688af2814 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.19 */ public class JRTConfigRequestV3Test extends JRTConfigRequestBase { @@ -71,7 +70,7 @@ public class JRTConfigRequestV3Test extends JRTConfigRequestBase { @Test public void emptypayload() { ConfigPayload payload = ConfigPayload.empty(); - SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, false, ConfigUtils.getMd5(payload)); + SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, 0, false, ConfigUtils.getMd5(payload)); serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5()); assertTrue(clientReq.validateResponse()); assertTrue(clientReq.hasUpdatedGeneration()); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java index e5fc5190ad1..c52245ecb35 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java @@ -15,7 +15,6 @@ import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.21 */ public class PayloadTest { diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java index f2d9edc7c35..22baab85c1c 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java @@ -21,9 +21,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.5 */ public class SlimeTraceSerializerTest { + @Test public void test_serializer() throws IOException { TraceNode root = new TraceNode(null, 1); diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java index 596983eebb8..4b191d85121 100644 --- a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java @@ -9,7 +9,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.3 */ public class TraceTest { @@ -58,4 +57,5 @@ public class TraceTest { assertTrue(trace2Str.contains("barbaz")); assertTrue(trace2Str.contains("quux")); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 92e7bf0300b..d3f37eb320e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -893,11 +893,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public void close() { metric.set(name, Duration.between(start, clock.instant()).toMillis(), - metric.createContext(Map.of("tenant", id.tenant().value(), - "application", id.application().value(), - "instance", id.instance().value(), - "environment", environment, - "region", region))); + metric.createContext(Map.of("applicationId", id.toFullString(), + "tenantName", id.tenant().value(), + "app", id.application().value() + "." + id.instance().value(), + "zone", environment + "." + region))); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 8b2c3e2cb0a..829a59a9598 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -134,6 +134,7 @@ public class ModelContextImpl implements ModelContext { private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; + private final boolean useNewAthenzFilter; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -166,6 +167,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.useNewAthenzFilter = Flags.USE_NEW_ATHENZ_FILTER.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -219,6 +222,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useBucketSpaceMetric() { return useBucketSpaceMetric; } + + @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 8c49d86243f..67549672408 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -59,8 +59,7 @@ public class TesterClient { } private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { - // TODO: Change log level to DEBUG - logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString()); + logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString()); try { return new ProxyResponse(httpClient.execute(request)); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java index bb752f9b804..0014d66026b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 34c7dced404..197b90e322c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -6,7 +6,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 67aa49ea0b9..48580dbc6f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -167,7 +167,7 @@ class GetConfigProcessor implements Runnable { log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); - ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5); + ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5); request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5()); respond(request); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java index dcbc21e536c..5235a2bcadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java @@ -29,7 +29,7 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory { String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes())); - return new SlimeConfigResponse(compressed, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(compressed, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java index 91809f90a70..bd0b117c3db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java @@ -25,7 +25,7 @@ public class UncompressedConfigResponseFactory implements ConfigResponseFactory Utf8Array rawPayload = payload.toUtf8Array(true); String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); - return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(rawPayload, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 1a41c1efd7a..49f59773bca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java index 65ebb38c2d0..97179e5234b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java index 6500449e557..8e51ac424f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a963252d7ca..0e076d60d52 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -342,11 +342,10 @@ public class ApplicationRepositoryTest { new MockTesterClient(), actual); deployApp(testAppLogServerWithContainer); - Map<String, ?> context = Map.of("tenant", "test1", - "application", "testapp", - "instance", "default", - "environment", "prod", - "region", "default"); + Map<String, ?> context = Map.of("applicationId", "test1.testapp.default", + "tenantName", "test1", + "app", "testapp.default", + "zone", "prod.default"); MockMetric expected = new MockMetric(); expected.set("deployment.prepareMillis", 0L, expected.createContext(context)); expected.set("deployment.activateMillis", 0L, expected.createContext(context)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java index db687857dae..c835bb29ec0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java @@ -15,70 +15,40 @@ public class MockTesterClient extends TesterClient { @Override public HttpResponse getStatus(String testerHostname, int port) { - return new MockStatusResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); + } + }; } @Override public HttpResponse getLog(String testerHostname, int port, Long after) { - return new MockLogResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log".getBytes(StandardCharsets.UTF_8)); + } + }; } @Override public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) { - return new MockStartTestsResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) { } + }; } @Override public HttpResponse isTesterReady(String testerHostname, int port) { - return new MockTesterReadyResponse(); - } - - private static class MockStatusResponse extends HttpResponse { - - private MockStatusResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); - } - - } - - private static class MockLogResponse extends HttpResponse { - - private MockLogResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("log".getBytes(StandardCharsets.UTF_8)); - } - - } - - private static class MockStartTestsResponse extends HttpResponse { - - private MockStartTestsResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) { } - - } - - private static class MockTesterReadyResponse extends HttpResponse { - - private MockTesterReadyResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) { } - + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("{ \"message\": \"OK\" } ".getBytes(StandardCharsets.UTF_8)); + } + }; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java index f2ee7815df3..9a18570db2d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java @@ -27,7 +27,6 @@ public class ServerCacheTest { private static String configMd5 = "mymd5"; private static String configMd5_2 = "mymd5_2"; private static ConfigDefinition def = new ConfigDefinition("mypayload", new String[0]); - private static ConfigDefinition def_2 = new ConfigDefinition("otherpayload", new String[0]); private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar"); private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz"); @@ -49,9 +48,9 @@ public class ServerCacheTest { cache = new ServerCache(new TestConfigDefinitionRepo(), userConfigDefinitionRepo); - cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def_2.getCNode(), 2, false, configMd5_2), configMd5_2); + cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5_2), configMd5_2); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java index 405fff3e190..ad910c2afc2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -4,14 +4,15 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ModelConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; @@ -33,12 +34,14 @@ import org.junit.Before; import org.junit.Test; import org.xml.sax.SAXException; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.util.List; +import java.nio.charset.StandardCharsets; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -111,22 +114,27 @@ public class ApplicationTest { } @Test - public void require_that_build_config_can_be_resolved() { - List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload(); - assertTrue(payload.get(1).contains("host")); + public void require_that_build_config_can_be_resolved() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, + ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\"")); } @Test - public void require_that_non_existent_fields_in_schema_is_skipped() { + public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException { // Ask for config without schema and check that we get correct default value back - List<String> payload = handler.resolveConfig(createSimpleConfigRequest()).getLegacyPayload(); - assertThat(payload.get(0), is("boolval false")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createSimpleConfigRequest()).serialize(baos, CompressionType.UNCOMPRESSED);; + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(StandardCharsets.UTF_8)); // Ask for config with wrong schema String[] schema = new String[1]; schema[0] = "boolval bool default=true"; // changed to be true, original is false - payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload(); - assertThat(payload.size(), is(1)); - assertThat(payload.get(0), is("boolval true")); + baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset())); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 71ae6955e56..1f034a92cbb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import org.junit.Before; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java index 33193bafc66..837cbd4605a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java @@ -11,8 +11,10 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -24,7 +26,7 @@ import java.util.function.Function; */ public class OrchestratorMock implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> hostInfos = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -34,14 +36,13 @@ public class OrchestratorMock implements Orchestrator { @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = hostInfos.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostInfo>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)) - .map(status -> status.isSuspended() ? HostInfo.createSuspended(status, Instant.EPOCH) - : HostInfo.createNoRemarks()); + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -49,12 +50,12 @@ public class OrchestratorMock implements Orchestrator { @Override public void resume(HostName hostName) { - suspendedHosts.remove(hostName); + hostInfos.remove(hostName); } @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java index 9a371639f52..ef0a5f6113d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java @@ -2,34 +2,25 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.ConfigResponse; - import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import org.junit.Test; import java.io.IOException; -import java.io.StringReader; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class HttpConfigResponseTest { @Test public void require_that_response_is_created_from_config() throws IOException { final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); - ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"); + ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"); HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 3ae98c1b8f2..089b662b797 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -2,28 +2,27 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.config.provision.ApplicationId; - +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import org.junit.Before; import org.junit.Test; + import java.io.IOException; -import java.io.StringReader; import java.util.Collections; import java.util.HashSet; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST; +import static com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -48,8 +47,7 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -75,16 +73,10 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index fb09aa99039..b72785876bc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -1,37 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; -import java.io.IOException; -import java.io.StringReader; -import java.util.Collections; -import java.util.HashSet; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.junit.Before; -import org.junit.Test; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; +import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class HttpGetConfigHandlerTest { @@ -58,9 +57,8 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -71,11 +69,10 @@ public class HttpGetConfigHandlerTest { "/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid"; final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder() .tenant(tenant) .applicationName("myapplication").instanceName("myinstance").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -121,18 +118,11 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 086dfa5d0d3..0b33de2a42c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -1,18 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; -import com.google.common.base.Joiner; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadApplier; @@ -24,12 +21,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.util.ConfigUtils; - import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; @@ -37,11 +33,14 @@ import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import java.io.IOException; -import java.io.StringReader; import java.util.Optional; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -161,11 +160,7 @@ public class RpcServerTest { builder.intval(123); SimpletypesConfig responseConfig = new SimpletypesConfig(builder); ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); - InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, - new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))) - .getTree(); return SlimeConfigResponse.fromConfigPayload(responsePayload, - targetDef, 3L, true, /* internalRedeploy */ ConfigUtils.getMd5(responsePayload)); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 91c0e4cc6bf..fe8fb91d71f 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -11,6 +11,23 @@ ], "fields": [] }, + "com.yahoo.container.handler.ClustersStatus$Require": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.container.handler.ClustersStatus$Require[] values()", + "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE", + "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL" + ] + }, "com.yahoo.container.handler.ClustersStatus": { "superClass": "com.yahoo.component.AbstractComponent", "interfaces": [], @@ -23,7 +40,8 @@ "public void setReceiveTrafficByDefault(boolean)", "public void setUp(java.lang.Object)", "public void setDown(java.lang.Object)", - "public boolean containerShouldReceiveTraffic()" + "public boolean containerShouldReceiveTraffic()", + "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)" ], "fields": [] }, diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index aab13a1cc7b..0ed0daa2141 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent { @Inject public ClustersStatus() { } + public enum Require {ONE, ALL} + /** Are there any (in-service influencing) clusters in this container? */ private boolean containerHasClusters; @@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent { setDown((String) clusterIdentifier); } - /** Returns whether this container should receive traffic based on the state of this */ + @Deprecated // TODO: Remove on Vespa 8 public boolean containerShouldReceiveTraffic() { + return containerShouldReceiveTraffic(Require.ONE); + } + /** + * Returns whether this container should receive traffic based on the state of this + * @param require Requirement for being up, ALL or ONE. + */ + public boolean containerShouldReceiveTraffic(Require require) { synchronized (mutex) { if (containerHasClusters) { - // Should receive traffic when at least one cluster is up - return clusterStatus.values().stream().anyMatch(status -> status==true); + switch (require) { + case ONE: + // Should receive traffic when at least one cluster is up + return clusterStatus.values().stream().anyMatch(status -> status == true); + case ALL: + default: + return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true); + } } else { return true; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index b9ef1627ce7..f712690efc5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -102,10 +102,15 @@ public class VipStatus { private void updateCurrentlyInRotation() { synchronized (mutex) { - if (rotationOverride != null) + if (rotationOverride != null) { currentlyInRotation = rotationOverride; - else - currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + } else { + if (healthState.status() == StateMonitor.Status.up) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE); + } else { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL); + } + } // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' if (currentlyInRotation) diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index f690c240537..faa08402cdc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -14,6 +14,7 @@ import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** @@ -27,7 +28,7 @@ public class StateMonitor extends AbstractComponent { private final static Logger log = Logger.getLogger(StateMonitor.class.getName()); - public enum Status {up, down, initializing}; + public enum Status {up, down, initializing} private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); private final Thread thread; @@ -37,6 +38,7 @@ public class StateMonitor extends AbstractComponent { private volatile MetricSnapshot snapshot; private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); + private final AtomicBoolean stopped = new AtomicBoolean(false); /** For testing */ public StateMonitor() { @@ -53,10 +55,16 @@ public class StateMonitor extends AbstractComponent { } StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { + this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)), + Status.valueOf(config.initialStatus()), + timer, threadFactory); + } + /* For Testing */ + public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) { this.timer = timer; - this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); + this.snapshotIntervalMs = snapshotIntervalMS; this.lastSnapshotTimeMs = timer.currentTimeMillis(); - this.status = Status.valueOf(config.initialStatus()); + this.status = status; thread = threadFactory.newThread(this::run); thread.start(); } @@ -99,13 +107,13 @@ public class StateMonitor extends AbstractComponent { private void run() { log.finest("StateMonitor started."); try { - while (!Thread.interrupted()) { - checkTime(); - Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + synchronized (stopped) { + while (!stopped.get()) { + checkTime(); + stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis()); + } } - } catch (InterruptedException e) { - - } + } catch (InterruptedException e) { } log.finest("StateMonitor stopped."); } @@ -137,12 +145,13 @@ public class StateMonitor extends AbstractComponent { @Override public void deconstruct() { - thread.interrupt(); + synchronized (stopped) { + stopped.set(true); + stopped.notifyAll(); + } try { thread.join(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + } catch (InterruptedException e) { } if (thread.isAlive()) { log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); } diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def index 5e70c72ae3f..4e91d85b2b8 100644 --- a/container-core/src/main/resources/configdefinitions/health-monitor.def +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -6,4 +6,4 @@ namespace=container.jdisc.config snapshot_interval double default=300 # Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values -initialStatus string default="up" +initialStatus string default="initializing" diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 52679c15957..d3479936544 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -4,6 +4,8 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.jdisc.state.StateMonitor; +import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; /** @@ -12,45 +14,79 @@ import org.junit.Test; * @author steinar */ public class VipStatusTestCase { + private static final String [] clusters = {"cluster1", "cluster2", "cluster3"}; - @Test - public void testVipStatusWorksWithClusters() { + private static QrSearchersConfig getSearchersCfg() { var b = new QrSearchersConfig.Builder(); var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name("cluster1"); - searchClusterB.name("cluster2"); - searchClusterB.name("cluster3"); + for (String cluster : clusters) { + searchClusterB.name(cluster); + } b.searchcluster(searchClusterB); - VipStatus v = new VipStatus(b.build()); + return b.build(); + } + private static VipStatus getVipStatus(StateMonitor.Status startState) { + return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> { + Thread thread = new Thread(runnable, "StateMonitor"); + thread.setDaemon(true); + return thread; + })); + } - String cluster1 = "cluster1"; - String cluster2 = "cluster2"; - String cluster3 = "cluster3"; + private static void removeAll(VipStatus v) { + for (String s : clusters) { + v.removeFromRotation(s); + } + } + private static void addAll(VipStatus v) { + for (String s : clusters) { + v.addToRotation(s); + } + } + private static void verifyUpOrDown(StateMonitor.Status status) { + VipStatus v = getVipStatus(status); + removeAll(v); // initial state assertFalse(v.isInRotation()); - - // one cluster becomes up - v.addToRotation(cluster1); + v.addToRotation(clusters[0]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[1]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[2]); assertTrue(v.isInRotation()); + } - // all clusters down - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + @Test + public void testInitializingOrDownRequireAllUp() { + verifyUpOrDown(StateMonitor.Status.initializing); + verifyUpOrDown(StateMonitor.Status.down); + } + + @Test + public void testUpRequireAllDown() { + VipStatus v = getVipStatus(StateMonitor.Status.initializing); assertFalse(v.isInRotation()); - // some clusters down - v.addToRotation(cluster2); + addAll(v); assertTrue(v.isInRotation()); - // all clusters up - v.addToRotation(cluster1); - v.addToRotation(cluster3); + + v.removeFromRotation(clusters[0]); + assertTrue(v.isInRotation()); + v.removeFromRotation(clusters[1]); assertTrue(v.isInRotation()); - // and down again - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + v.removeFromRotation(clusters[2]); + assertFalse(v.isInRotation()); // All down + v.addToRotation(clusters[1]); assertFalse(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); // All up + v.removeFromRotation(clusters[0]); + v.removeFromRotation(clusters[2]); + assertTrue(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); } } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index 78541137db5..8a1640e2c0e 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -57,7 +57,8 @@ public class StateHandlerTestBase { HealthMonitorConfig healthMonitorConfig = new HealthMonitorConfig( new HealthMonitorConfig.Builder() - .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL))); + .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)) + .initialStatus("up")); ThreadFactory threadFactory = ignored -> mock(Thread.class); this.monitor = new StateMonitor(healthMonitorConfig, timer, threadFactory); builder.guiceModules().install(new AbstractModule() { diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index 35280e12146..81de014c6ad 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -14,6 +14,8 @@ "public abstract javax.net.ssl.SSLContext getRoleSslContext(java.lang.String, java.lang.String)", "public abstract java.lang.String getRoleToken(java.lang.String)", "public abstract java.lang.String getRoleToken(java.lang.String, java.lang.String)", + "public abstract java.lang.String getAccessToken(java.lang.String)", + "public abstract java.lang.String getAccessToken(java.lang.String, java.util.List)", "public abstract java.util.List getIdentityCertificate()", "public abstract java.security.PrivateKey getPrivateKey()" ], @@ -31,6 +33,17 @@ ], "fields": [] }, + "com.yahoo.container.jdisc.secretstore.SecretNotFoundException": { + "superClass": "java.lang.RuntimeException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)" + ], + "fields": [] + }, "com.yahoo.container.jdisc.secretstore.SecretStore": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java index 9045db3eda5..bb862aeca82 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java @@ -59,6 +59,16 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr } @Override + public String getAccessToken(String domain) { + throw new UnsupportedOperationException(message); + } + + @Override + public String getAccessToken(String domain, List<String> roles) { + throw new UnsupportedOperationException(message); + } + + @Override public List<X509Certificate> getIdentityCertificate() { throw new UnsupportedOperationException(message); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index 1db3f4a3b42..696aab85b0c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -16,6 +16,8 @@ public interface AthenzIdentityProvider { SSLContext getRoleSslContext(String domain, String role); String getRoleToken(String domain); String getRoleToken(String domain, String role); + String getAccessToken(String domain); + String getAccessToken(String domain, List<String> roles); List<X509Certificate> getIdentityCertificate(); PrivateKey getPrivateKey(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java new file mode 100644 index 00000000000..b9439432c06 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.secretstore; + +/** + * @author mortent + */ +public class SecretNotFoundException extends RuntimeException { + + public SecretNotFoundException(String message) { + super(message); + } +} diff --git a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java index 00272778a2b..7ca5d2a6b10 100644 --- a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java +++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java @@ -72,6 +72,7 @@ public class GUIHandlerTest { private String servicesXml() { return "<container version='1.0'>\n" + + " <accesslog type='disabled'/>\n" + " <handler id='com.yahoo.search.query.gui.GUIHandler'>\n" + " <binding>http://*/querybuilder/*</binding>\n" + " </handler>\n" + diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b5fbe235c43..437db99c45b 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1954,6 +1954,7 @@ "methods": [ "public void <init>(com.yahoo.search.cluster.NodeManager)", "public void <init>(com.yahoo.search.cluster.NodeManager, boolean)", + "public void start()", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public void add(java.lang.Object, boolean)", "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)", @@ -1978,8 +1979,8 @@ "methods": [ "public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)", "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)", - "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", - "public final void ping(java.lang.Object, java.util.concurrent.Executor)", + "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", + "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)", "protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)", "public final com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)", @@ -2074,7 +2075,8 @@ "methods": [ "public abstract void working(java.lang.Object)", "public abstract void failed(java.lang.Object)", - "public abstract void ping(java.lang.Object, java.util.concurrent.Executor)", + "public void ping(java.lang.Object, java.util.concurrent.Executor)", + "public void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "public void pingIterationCompleted()" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 65d5879b004..365ee299ca4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -26,6 +26,7 @@ import java.util.Set; public class Index { public static class Attribute { + private boolean tokenizedContent = false; public final String name; @@ -207,20 +208,12 @@ public class Index { } } - /** - * Whether terms in this field are lower cased when indexing. - * - * @param lowercase true if terms are lowercased - */ + /** Sets whether terms in this field are lowercased when indexing. */ public void setLowercase(boolean lowercase) { this.lowercase = lowercase; } - /** - * Whether terms in this field are lower cased when indexing. - * - * @return true if terms are lowercased - */ + /** Returns whether terms in this field are lowercased when indexing. */ public boolean isLowercase() { return lowercase; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java index bac227ac3e3..55e16804602 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java @@ -31,14 +31,17 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { } } + @Override public ItemType getItemType() { return ItemType.AND; } + @Override public String getName() { return "SAND"; } + @Override public String getIndexName() { if (getItemCount() == 0) { return ""; @@ -54,4 +57,5 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { i.next().setWeight(w); } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java index 13673144a0a..d0ffcd2d0e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java @@ -3,10 +3,9 @@ package com.yahoo.prelude.query; /** - * An interface used for anything which represents a single block - * of query input. + * An interface used for anything which represents a single block of query input. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface BlockItem extends HasIndexItem { @@ -39,4 +38,5 @@ public interface BlockItem extends HasIndexItem { * is necessary to change operator? */ SegmentingRule getSegmentingRule(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java index a06009e642a..300d40d4366 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java @@ -79,4 +79,5 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements super.disclose(discloser); discloser.addProperty("index", index); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index ee4c0d4d9f0..5e292a06b0f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -520,7 +520,7 @@ abstract class StructuredParser extends AbstractParser { /** Returns a word, a phrase, or another composite */ private Item phraseBody(String indexName) { boolean quoted = false; - PhraseItem phrase = null; + CompositeItem composite = null; Item firstWord = null; boolean starAfterFirst = false; boolean starBeforeFirst; @@ -539,7 +539,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord(indexName, (firstWord != null) || (phrase != null)); + Item word = phraseWord(indexName, (firstWord != null) || (composite != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { @@ -555,34 +555,37 @@ abstract class StructuredParser extends AbstractParser { ((PhraseSegmentItem) word).setExplicit(true); } - if (phrase != null) { - phrase.addItem(word); + if (composite != null) { + composite.addItem(word); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); if (submodes.site) uriItem.setEndAnchorDefault(true); - phrase = uriItem; + composite = uriItem; } else { - phrase = new PhraseItem(); + if (quoted || indexFacts.getIndex(indexName).getPhraseSegmenting()) + composite = new PhraseItem(); + else + composite = new AndItem(); } - if (quoted || submodes.site || submodes.url) { - phrase.setExplicit(true); + if ( (quoted || submodes.site || submodes.url) && composite instanceof PhraseItem) { + ((PhraseItem)composite).setExplicit(true); } if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); - phrase.addItem(word); + composite.addItem(firstWord); + composite.addItem(word); } else if (word instanceof PhraseItem) { - phrase = (PhraseItem) word; + composite = (PhraseItem)word; } else { firstWord = word; starAfterFirst = tokens.skipNoIgnore(STAR); @@ -609,29 +612,29 @@ abstract class StructuredParser extends AbstractParser { braceLevelURL = 0; - if (phrase != null) { + if (composite != null) { if (addEndMarking()) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - return phrase; + return composite; } else if (firstWord != null && submodes.site) { if (starAfterFirst && !addStartOfHostMarker) { return firstWord; } else { - phrase = new PhraseItem(); + composite = new PhraseItem(); + ((PhraseItem)composite).setExplicit(true); if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); + composite.addItem(firstWord); if (!starAfterFirst) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - phrase.setExplicit(true); - return phrase; + return composite; } } else { if (firstWord != null && firstWord instanceof TermItem && (starAfterFirst || starBeforeFirst)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index fdd6ad47a98..ce13045b518 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -111,25 +111,17 @@ public class NormalizingSearcher extends Searcher { } private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) { - if (!block.isNormalizable()) { - return; - } - { - Index index = indexFacts.getIndex(block.getIndexName()); - if (index.isAttribute()) { - return; - } - if (!index.getNormalize()) { - return; - } - } + if ( ! block.isNormalizable()) return; + + Index index = indexFacts.getIndex(block.getIndexName()); + if (index.isAttribute()) return; + if ( ! index.getNormalize()) return; List<Alternative> terms = block.getAlternatives(); for (Alternative term : terms) { String accentDropped = linguistics.getTransformer().accentDrop(term.word, language); - if (!term.word.equals(accentDropped) && accentDropped.length() > 0) { + if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0) block.addTerm(accentDropped, term.exactness * .7d); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java index 84c793a6df1..5a936d42ccc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java @@ -4,6 +4,8 @@ package com.yahoo.prelude.querytransform; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.HasIndexItem; +import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NotItem; @@ -169,7 +171,9 @@ public class QueryRewrite { removeOtherNonrankedChildren(item, i); recall = Recall.RECALLS_EVERYTHING; } else if ((item instanceof AndItem) || (item instanceof NearItem)) { - item.removeItem(i); + if ( ! isRanked(item.getItem(i))) { + item.removeItem(i); + } } else if (item instanceof RankItem) { // empty } else { @@ -200,6 +204,20 @@ public class QueryRewrite { parent.removeItem(i); } } + + private static boolean isRanked(Item item) { + if (item instanceof CompositeItem) { + for (Item child : ((CompositeItem)item).items()) + if (isRanked(child)) return true; + return false; + } + else if (item instanceof HasIndexItem && Hit.SDDOCNAME_FIELD.equals(((HasIndexItem)item).getIndexName())) { + return false; // No point in ranking by sddocname + } + else { + return item.isRanked(); + } + } private static Item collapseSingleComposites(Item item) { if (!(item instanceof CompositeItem)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 655fbf6acc3..9a9044def2d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -188,13 +188,10 @@ public class StemmingSearcher extends Searcher { return (Item) w; } - if (context.isCJK) { - composite = chooseCompositeForCJK(current, - ((Item) current).getParent(), - indexName); - } else { - composite = phraseSegment(current, indexName); - } + if (context.isCJK) + composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName); + else + composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); @@ -331,39 +328,34 @@ public class StemmingSearcher extends Searcher { } } + private CompositeItem chooseComposite(BlockItem current, CompositeItem parent, String indexName) { + if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) + return createPhraseSegment(current, indexName); + else + return createAndSegment(current); + + } + private CompositeItem chooseCompositeForCJK(BlockItem current, CompositeItem parent, String indexName) { - CompositeItem composite; - if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) { - if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) { - composite = phraseSegment(current, indexName); - } else - composite = createAndSegment(current); - } else { - switch (current.getSegmentingRule()) { - case PHRASE: - composite = phraseSegment(current, indexName); - break; - case BOOLEAN_AND: - composite = createAndSegment(current); - break; + if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) + return chooseComposite(current, parent, indexName); + + switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only + case PHRASE: return createPhraseSegment(current, indexName); + case BOOLEAN_AND: return createAndSegment(current); default: - throw new IllegalArgumentException( - "Unknown segmenting rule: " - + current.getSegmentingRule() - + ". This is a bug in Vespa, as the implementation has gotten out of sync." - + " Please create a ticket as soon as possible."); - } + throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + + ". This is a bug in Vespa, as the implementation has gotten out of sync." + + " Please create a ticket as soon as possible."); } - return composite; } private AndSegmentItem createAndSegment(BlockItem current) { return new AndSegmentItem(current.stringValue(), true, true); } - private CompositeItem phraseSegment(BlockItem current, String indexName) { - CompositeItem composite; - composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); + private CompositeItem createPhraseSegment(BlockItem current, String indexName) { + CompositeItem composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); composite.setIndexName(indexName); return composite; } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 395d8853603..1e3f11f4f78 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -288,7 +288,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(""); } - /** * Construct a query from a string formatted in the http style, e.g <code>?query=test&offset=10&hits=13</code> * The query must be uri encoded. @@ -297,7 +296,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(query, null); } - /** * Creates a query from a request * diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index d4b6279be89..55f0816514d 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -38,6 +38,9 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); + /** @deprecated It is not advised to start the monitoring thread in the constructor. + * Use ClusterMonitor(NodeManager manager, false) and explicit start(). */ + @Deprecated public ClusterMonitor(NodeManager<T> manager) { this(manager, true); } @@ -50,6 +53,12 @@ public class ClusterMonitor<T> { } } + public void start() { + if ( ! monitorThread.isAlive()) { + monitorThread.start(); + } + } + /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } @@ -101,7 +110,7 @@ public class ClusterMonitor<T> { public void ping(Executor executor) { for (Iterator<BaseNodeMonitor<T>> i = nodeMonitorIterator(); i.hasNext() && !closed.get(); ) { BaseNodeMonitor<T> monitor= i.next(); - nodeManager.ping(monitor.getNode(), executor); // Cause call to failed or responded + nodeManager.ping(this, monitor.getNode(), executor); // Cause call to failed or responded } if (closed.get()) return; // Do nothing to change state if close has started. nodeManager.pingIterationCompleted(); diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 20f56c86f7b..2d05168731a 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -58,7 +58,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod this(id, connections, hasher, internal, true); } - public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { + protected ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { super(id); this.hasher = hasher; this.monitor = new ClusterMonitor<>(this, startPingThread); @@ -70,7 +70,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override - public final void ping(T p, Executor executor) { + public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { log(LogLevel.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask<Pong> future = new FutureTask<>(pinger); @@ -80,7 +80,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod Throwable logThrowable = null; try { - pong = future.get(monitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); + pong = future.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { pong = new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + p)); logThrowable = e; @@ -96,10 +96,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod future.cancel(true); if (pong.badResponse()) { - monitor.failed(p, pong.error().get()); + clusterMonitor.failed(p, pong.error().get()); log(LogLevel.FINE, "Failed ping - ", pong); } else { - monitor.responded(p); + clusterMonitor.responded(p); log(LogLevel.FINE, "Answered ping - ", p); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index 226e0180d2e..7b10992dff8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -30,7 +30,7 @@ public class MonitorConfiguration { * The number of milliseconds to attempt to complete a request * before giving up */ - private long requestTimeout = 5000; + private final long requestTimeout = 5000; /** * The number of milliseconds a node is allowed to fail before we diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 9b20139e3c5..481f1e1b5a5 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -19,9 +19,21 @@ public interface NodeManager<T> { /** * Called when a node should be pinged. - * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. */ - void ping(T node, Executor executor); + @Deprecated + default void ping(T node, Executor executor) { + throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); + } + + /** + * Called when a node should be pinged. + * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded + */ + default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) { + ping(node, executor); + } /** Called right after a ping has been issued to each node. This default implementation does nothing. */ default void pingIterationCompleted() {} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 03b51fbaf70..91bd5c6da11 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -11,8 +11,10 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; @@ -58,6 +60,7 @@ public class Dispatcher extends AbstractComponent { /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; + private final ClusterMonitor clusterMonitor; private final LoadBalancer loadBalancer; @@ -87,44 +90,48 @@ public class Dispatcher extends AbstractComponent { ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus), - dispatchConfig, - metric); + this(new RpcResourcePool(dispatchConfig), clusterId, dispatchConfig, clusterInfoConfig, vipStatus, metric); } - private Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { - this(searchCluster, - dispatchConfig, - new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster), - metric); + private Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, + DispatchConfig dispatchConfig, + ClusterInfoConfig clusterInfoConfig, + VipStatus vipStatus, + Metric metric) { + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), + vipStatus, new RpcPingFactory(resourcePool)), + dispatchConfig, metric); + } - /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, - DispatchConfig dispatchConfig, - RpcInvokerFactory rcpInvokerFactory, - Metric metric) { - this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric); + private Dispatcher(RpcResourcePool resourcePool, SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { + this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric); } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, + protected Dispatcher(ClusterMonitor clusterMonitor, + SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, - PingFactory pingFactory, Metric metric) { if (dispatchConfig.useMultilevelDispatch()) throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported"); this.searchCluster = searchCluster; + this.clusterMonitor = clusterMonitor; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); - - searchCluster.startClusterMonitoring(pingFactory); + searchCluster.addMonitoring(clusterMonitor); + try { + while ( ! searchCluster.hasInformationAboutAllNodes()) { + Thread.sleep(1); + } + } catch (InterruptedException e) {} } /** Returns the search cluster this dispatches to */ @@ -134,8 +141,8 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The seach cluster must be shutdown first as it uses the invokerfactory. */ - searchCluster.shutDown(); + /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + clusterMonitor.shutdown(); invokerFactory.release(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 5c9928de924..a45ec59c3ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -14,6 +14,8 @@ import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; @@ -22,7 +24,7 @@ import java.util.concurrent.Callable; /** * @author ollivir */ -public class RpcInvokerFactory extends InvokerFactory implements PingFactory { +public class RpcInvokerFactory extends InvokerFactory { /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); @@ -63,9 +65,4 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { public void release() { rpcResourcePool.release(); } - - @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - return new RpcPing(node, monitor, rpcResourcePool); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index e0f1dc5e675..5e04f1d7a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -10,55 +10,63 @@ import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.rpc.Client.ResponseOrError; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.result.ErrorMessage; import com.yahoo.yolean.Exceptions; -import java.util.concurrent.Callable; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; -public class RpcPing implements Callable<Pong> { +public class RpcPing implements Pinger, Client.ResponseReceiver { + private static final Logger log = Logger.getLogger(RpcPing.class.getName()); private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; private final Node node; private final RpcResourcePool resourcePool; private final ClusterMonitor<Node> clusterMonitor; + private final long pingSequenceId; + private final PongHandler pongHandler; - public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) { + public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool, PongHandler pongHandler) { this.node = node; this.resourcePool = rpcResourcePool; this.clusterMonitor = clusterMonitor; + pingSequenceId = node.createPingSequenceId(); + this.pongHandler = pongHandler; } @Override - public Pong call() throws Exception { + public void ping() { try { - var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1); - - sendPing(queue); + sendPing(); + } catch (RuntimeException e) { + pongHandler.handle(new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + + ": " + Exceptions.toMessageString(e)))); + } + } - var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS); - if (responseOrError == null) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); - } else if (responseOrError.error().isPresent()) { - return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); - } + private Pong toPong(ResponseOrError<ProtobufResponse> responseOrError) { + if (responseOrError == null) { + return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); + } else if (responseOrError.error().isPresent()) { + return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); + } + try { return decodeReply(responseOrError.response().get()); - } catch (RuntimeException e) { - return new Pong( - ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e))); + } catch (InvalidProtocolBufferException e) { + return new Pong(ErrorMessage.createBackendCommunicationError(e.getMessage())); } } - private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) { + private void sendPing() { var connection = resourcePool.getConnection(node.key()); var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -76,4 +84,14 @@ public class RpcPing implements Callable<Pong> { } } + @Override + public void receive(ResponseOrError<ProtobufResponse> response) { + if (node.isLastReceivedPong(pingSequenceId)) { + pongHandler.handle(toPong(response)); + } else { + //TODO Reduce to debug or remove once we have enumerated what happens here. + log.info("Pong " + pingSequenceId + " from node " + node.key() + " in group " + node.group() + + " with hostname " + node.hostname() + " received too late, latest is " + node.getLastReceivedPongId()); + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java new file mode 100644 index 00000000000..ac8f0a59c20 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -0,0 +1,18 @@ +package com.yahoo.search.dispatch.rpc; + +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; + +public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { + this.rpcResourcePool = rpcResourcePool; + } + @Override + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { + return new RpcPing(node, monitor, rpcResourcePool, pongHandler); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 2f70c37cd48..e93b633f09d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -21,6 +21,8 @@ public class Node { private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicLong pingSequence = new AtomicLong(0); + private final AtomicLong lastPong = new AtomicLong(0); public Node(int key, String hostname, int group) { this.key = key; @@ -28,6 +30,18 @@ public class Node { this.group = group; } + /** Give a monotonically increasing sequence number.*/ + public long createPingSequenceId() { return pingSequence.incrementAndGet(); } + /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/ + public boolean isLastReceivedPong(long pingId ) { + long last = lastPong.get(); + while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) { + last = lastPong.get(); + } + return last < pingId; + } + public long getLastReceivedPongId() { return lastPong.get(); } + /** Returns the unique and stable distribution key of this node */ public int key() { return key; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index b16fa941f68..2e07d8d61e6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -1,13 +1,11 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; -import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; -import java.util.concurrent.Callable; public interface PingFactory { - Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); + Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java new file mode 100644 index 00000000000..b4a7ccbf98c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -0,0 +1,12 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +/** + * Send a ping and ensure that the pong is propagated to the ponghandler. + * Should not wait as this should be done in parallel on all nodes. + * + * @author baldersheim + */ +public interface Pinger { + void ping(); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java new file mode 100644 index 00000000000..c0579b5d36e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -0,0 +1,12 @@ +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.prelude.Pong; + +/** + * Handle the Pong result of a Ping. + * + * @author baldersheim + */ +public interface PongHandler { + void handle(Pong pong); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 5f211c37917..d462479226a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -10,7 +10,6 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; -import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.LinkedHashMap; @@ -18,13 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -43,9 +36,8 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; private final ImmutableList<Group> orderedGroups; - private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; - private PingFactory pingFactory; + private final PingFactory pingFactory; private long nextLogTime = 0; /** @@ -58,10 +50,12 @@ public class SearchCluster implements NodeManager<Node> { */ private final Optional<Node> localCorpusDispatchTarget; - public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, + VipStatus vipStatus, PingFactory pingFactory) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; this.vipStatus = vipStatus; + this.pingFactory = pingFactory; List<Node> nodes = toNodes(dispatchConfig); this.size = nodes.size(); @@ -84,29 +78,18 @@ public class SearchCluster implements NodeManager<Node> { this.nodesByHost = nodesByHostBuilder.build(); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), - size, - containerClusterSize, - nodesByHost, - groups); - - this.clusterMonitor = new ClusterMonitor<>(this); - } - - public void shutDown() { - clusterMonitor.shutdown(); + size, + containerClusterSize, + nodesByHost, + groups); } - - public void startClusterMonitoring(PingFactory pingFactory) { - this.pingFactory = pingFactory; - + public void addMonitoring(ClusterMonitor clusterMonitor) { for (var group : orderedGroups) { for (var node : group.nodes()) clusterMonitor.add(node, true); } } - ClusterMonitor<Node> clusterMonitor() { return clusterMonitor; } - private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname, int searchClusterSize, int containerClusterSize, @@ -247,7 +230,7 @@ public class SearchCluster implements NodeManager<Node> { vipStatus.removeFromRotation(clusterId); } - private boolean hasInformationAboutAllNodes() { + public boolean hasInformationAboutAllNodes() { return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); } @@ -263,24 +246,33 @@ public class SearchCluster implements NodeManager<Node> { return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id(); } + private static class PongCallback implements PongHandler { + private final ClusterMonitor<Node> clusterMonitor; + private final Node node; + PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { + this.node = node; + this.clusterMonitor = clusterMonitor; + } + @Override + public void handle(Pong pong) { + if (pong.badResponse()) { + clusterMonitor.failed(node, pong.error().get()); + } else { + if (pong.activeDocuments().isPresent()) { + node.setActiveDocuments(pong.activeDocuments().get()); + } + clusterMonitor.responded(node); + } + } + } + /** Used by the cluster monitor to manage node status */ @Override - public void ping(Node node, Executor executor) { + public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) { if (pingFactory == null) return; // not initialized yet - FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor)); - executor.execute(futurePong); - Pong pong = getPong(futurePong, node); - futurePong.cancel(true); - - if (pong.badResponse()) { - clusterMonitor.failed(node, pong.error().get()); - } else { - if (pong.activeDocuments().isPresent()) { - node.setActiveDocuments(pong.activeDocuments().get()); - } - clusterMonitor.responded(node); - } + Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor)); + pinger.ping(); } private void pingIterationCompletedSingleGroup() { @@ -353,20 +345,6 @@ public class SearchCluster implements NodeManager<Node> { return workingNodes + nodesAllowedDown >= nodesInGroup; } - private Pong getPong(FutureTask<Pong> futurePong, Node node) { - try { - return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + node)); - } catch (ExecutionException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + node)); - } catch (TimeoutException e) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out")); - } - } - /** * Calculate whether a subset of nodes in a group has enough coverage */ diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 0981c6e8dad..316b2376060 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -29,7 +29,7 @@ import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index ae50756331c..775dca7c444 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -45,8 +45,7 @@ import com.yahoo.search.yql.VespaGroupingStep; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index b250560e2f3..5d4f39cecbf 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -70,7 +70,7 @@ public class QueryProfileProperties extends Properties { * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile */ @Override - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { // TODO: Refactor try { name = unalias(name, context); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 30fc98ac6b1..4f30331e738 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -37,7 +37,7 @@ public class PropertyMap extends Properties { * Return true if this value should be set in this map, false if the set should be propagated instead * This default implementation always returns true. */ - protected boolean shouldSet(CompoundName name,Object value) { return true; } + protected boolean shouldSet(CompoundName name, Object value) { return true; } @Override public Object get(CompoundName name, Map<String,String> context, diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index a4c150b606e..dfe6c2af44b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -44,7 +44,7 @@ public class QueryProperties extends Properties { @Override public Object get(CompoundName key, - Map<String,String> context, + Map<String, String> context, com.yahoo.processing.request.Properties substitution) { if (key.size() == 2 && key.first().equals(Model.MODEL)) { Model model = query.getModel(); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index ee09521fa74..6cf27fc9a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -7,7 +7,7 @@ import com.yahoo.search.query.Properties; import java.util.Map; /** - * Turns get(name) into get(name,request) using the request given at construction time. + * Turns get(name) into get(name, request) using the request given at construction time. * This is used to allow the query's request to be supplied to all property requests * without forcing users of the query.properties() to supply this explicitly. * @@ -22,18 +22,18 @@ public class RequestContextProperties extends Properties { } @Override - public Object get(CompoundName name,Map<String,String> context, + public Object get(CompoundName name, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.get(name, context == null ? requestMap : context, substitution); } @Override - public void set(CompoundName name,Object value,Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String,String> context) { super.set(name, value, context == null ? requestMap : context); } @Override - public Map<String, Object> listProperties(CompoundName path,Map<String,String> context, + public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.listProperties(path, context == null ? requestMap : context, substitution); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 1e8f436a05a..25488aa7bbc 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -44,4 +44,5 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { Index index = indexFacts.getIndex(sb.toString()); return index.isLowercase() || index.isAttribute(); } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index b5e54b46e5a..0f49f6029ad 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -151,7 +151,8 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - Dispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); dispatch_1.deconstruct(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 440e3b8d78f..707b051c03b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -2,8 +2,10 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.container.handler.VipStatus; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; @@ -13,6 +15,8 @@ import java.util.List; class MockDispatcher extends Dispatcher { + public final ClusterMonitor clusterMonitor; + public static MockDispatcher create(List<Node> nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); @@ -20,18 +24,19 @@ class MockDispatcher extends Dispatcher { } public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, - int containerClusterSize, VipStatus vipStatus) { + int containerClusterSize, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus); - return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool); + var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus, new RpcPingFactory(rpcResourcePool)); + return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { - this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { + this(clusterMonitor, searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { - super(searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { + super(clusterMonitor, searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + this.clusterMonitor = clusterMonitor; } static DispatchConfig toDispatchConfig(List<Node> nodes) { diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java index 11922cf640a..36137abd9b8 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java @@ -1,15 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform.test; +import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.prelude.querytransform.RecallSearcher; import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -38,9 +45,17 @@ public class QueryRewriteTestCase { assertRewritten(query, "OR sddocname:per foo bar"); ((OrItem)query.getModel().getQueryTree().getRoot()).getItem(2).setRanked(false); // set 'bar' unranked assertRewritten(query, "OR sddocname:per foo"); - assertRewritten("sddocname:per OR foo OR (bar AND fuz)", "per", "OR sddocname:per foo (AND bar fuz)"); + } + @Test + public void testRankContributingTermsAreNotRemovedOnFullRecall() { + Query query = new Query(QueryTestCase.httpEncode("?query=default:term1 OR default:term2 OR default:term3 OR sddocname:per&type=adv&recall=+id:1&restrict=per")); + RecallSearcher searcher = new RecallSearcher(); + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertNull(result.hits().getError()); + assertNull(QueryCanonicalizer.canonicalize(query)); + assertRewritten(query, "AND (OR default:term1 default:term2 default:term3 sddocname:per) |id:1"); } @Test @@ -88,6 +103,7 @@ public class QueryRewriteTestCase { private static void assertRewritten(Query query, String expectedOptimizedQuery) { QueryRewrite.optimizeByRestrict(query); + QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString()); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index de6bafa267a..5433a28dd6e 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -1,22 +1,21 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.prelude.fastsearch.test.MockMetric; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import org.junit.Test; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.Callable; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.junit.Assert.assertEquals; @@ -39,9 +38,10 @@ public class DispatcherTest { assertEquals(2, nodes.get(0).key()); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(q, null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -53,9 +53,10 @@ public class DispatcherTest { } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -69,9 +70,10 @@ public class DispatcherTest { assertTrue(acceptIncompleteCoverage); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -80,8 +82,9 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 1); MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); disp.getSearchInvoker(new Query(), null); + disp.deconstruct(); fail("Expected exception"); } catch (IllegalStateException e) { @@ -142,7 +145,7 @@ public class DispatcherTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { fail("Unexpected call to createPinger"); return null; } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 0496194f8ed..6eedb8239a9 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index a976b287f63..3b4d58cdfc2 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, 1, null); + super(clusterId, dispatchConfig, 1, null, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 10a579b0e4f..766f9ea6c2d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -20,6 +20,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,6 +35,7 @@ public class SearchClusterTest { final int nodesPerGroup; final VipStatus vipStatus; final SearchCluster searchCluster; + final ClusterMonitor clusterMonitor; final List<AtomicInteger> numDocsPerNode; List<AtomicInteger> pingCounts; @@ -57,74 +59,76 @@ public class SearchClusterTest { numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } - searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, vipStatus); + searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, + vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); + clusterMonitor = new ClusterMonitor(searchCluster, false); + searchCluster.addMonitoring(clusterMonitor); } - void startMonitoring() { - searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); - } - - static private int maxFrom(List<AtomicInteger> list) { - int max = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() > max) { - max = v.get(); + private int maxPingCount() { + int max = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() > max) { + max = count.get(); } } return max; } - private static int minFrom(List<AtomicInteger> list) { - int min = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() < min) { - min = v.get(); + private int minPingCount() { + int min = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() < min) { + min = count.get(); } } return min; } - private void waitAtLeast(int atLeast, List<AtomicInteger> list) { - while (minFrom(list) < atLeast) { + void waitOneFullPingRound() { + int minPingCount = minPingCount(); + int atLeast = maxPingCount() + 1; + while (minPingCount < atLeast) { ExecutorService executor = Executors.newCachedThreadPool(); - searchCluster.clusterMonitor().ping(executor); + clusterMonitor.ping(executor); executor.shutdown(); try { boolean completed = executor.awaitTermination(120, TimeUnit.SECONDS); if ( ! completed ) throw new IllegalStateException("Ping thread timed out"); + // Since a separate thread will be modifying values in pingCounts, we need to wait for the thread to + // finish before re-reading the minimum value + minPingCount = minPingCount(); } catch (InterruptedException e) { - System.out.println("Ping thread interrupted"); + throw new RuntimeException(e); } } } - void waitOneFullPingRound() { - waitAtLeast(maxFrom(pingCounts) + 1, pingCounts); - } - @Override public void close() { - searchCluster.shutDown(); + clusterMonitor.shutdown(); } static class Factory implements PingFactory { - static class Pinger implements Callable<Pong> { + static class PingJob implements Pinger { private final AtomicInteger numDocs; private final AtomicInteger pingCount; - Pinger(AtomicInteger numDocs, AtomicInteger pingCount) { + private final PongHandler pongHandler; + PingJob(AtomicInteger numDocs, AtomicInteger pingCount, PongHandler pongHandler) { this.numDocs = numDocs; this.pingCount = pingCount; + this.pongHandler = pongHandler; } @Override - public Pong call() { + public void ping() { int docs = numDocs.get(); - pingCount.incrementAndGet(); - return (docs < 0) + pongHandler.handle ((docs < 0) ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs)) - : new Pong(docs); + : new Pong(docs)); + pingCount.incrementAndGet(); } } @@ -139,9 +143,9 @@ public class SearchClusterTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { int index = node.group() * numPerGroup + node.key(); - return new Pinger(activeDocs.get(index), pingCounts.get(index)); + return new PingJob(activeDocs.get(index), pingCounts.get(index), pongHandler); } } @@ -153,7 +157,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); } @@ -162,7 +165,6 @@ public class SearchClusterTest { @Test public void requireThatZeroDocsAreFine() { try (State test = new State("cluster.1", 2, "a", "b")) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -184,7 +186,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); } @@ -196,7 +197,6 @@ public class SearchClusterTest { assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); test.numDocsPerNode.get(0).set(-1); @@ -209,7 +209,6 @@ public class SearchClusterTest { public void requireThatVipStatusDownWhenLocalIsDown() { try (State test = new State("cluster.1",1,HostName.getLocalhost(), "b")) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); @@ -245,7 +244,6 @@ public class SearchClusterTest { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); @@ -273,8 +271,8 @@ public class SearchClusterTest { static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) { List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup); for (int g = 0; g < numGroups; g++) { - for (int n=0; n < nodesPerGroup; n++) { - nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString()); + for (int n = 0; n < nodesPerGroup; n++) { + nodeNames.add("node." + g + '.' + n); } } return nodeNames; @@ -284,7 +282,6 @@ public class SearchClusterTest { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { - test.startMonitoring(); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); @@ -310,4 +307,18 @@ public class SearchClusterTest { verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3); } + @Test + public void requireThatPingSequenceIsUpHeld() { + Node node = new Node(1, "n", 1); + assertEquals(1, node.createPingSequenceId()); + assertEquals(2, node.createPingSequenceId()); + assertEquals(0, node.getLastReceivedPongId()); + assertTrue(node.isLastReceivedPong(2)); + assertEquals(2, node.getLastReceivedPongId()); + assertFalse(node.isLastReceivedPong(1)); + assertFalse(node.isLastReceivedPong(2)); + assertTrue(node.isLastReceivedPong(3)); + assertEquals(3, node.getLastReceivedPongId()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 02e2152d7c9..272092b6fc0 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -12,7 +12,7 @@ import com.yahoo.net.HostName; import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.After; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 46efb736918..bda191ee910 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -326,14 +326,14 @@ public class QueryProfileTestCase { assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b")); } - /** Tests that dots are followed when setting overridability */ + /** Dots are followed when setting overridability */ @Test public void testInstanceOverridable() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); profile.set("model.defaultIndex","default", null); profile.setOverridable("model.defaultIndex", false,null); - assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); // Parameters should be ignored Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null)); @@ -345,7 +345,7 @@ public class QueryProfileTestCase { assertEquals("de", query.getModel().getLanguage().languageCode()); } - /** Tests that dots are followed when setting overridability...also with variants */ + /** Dots are followed when setting overridability, also with variants */ @Test public void testInstanceOverridableWithVariants() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ea34be5cf37..e71ed308aaf 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -889,7 +889,7 @@ public class QueryTestCase { @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -899,12 +899,12 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND myfield:'it s' myfield:\"a b\" myfield:c", query.getModel().getQueryTree().toString()); } @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -914,7 +914,7 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 6ca5cae0455..9e5b01a91d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -46,7 +46,7 @@ public interface ServiceRegistry { Mailer mailer(); - ApplicationCertificateProvider applicationCertificateProvider(); + EndpointCertificateProvider endpointCertificateProvider(); MeteringClient meteringService(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index ec60f11060d..53e80fcb2ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; @@ -74,6 +75,16 @@ public class ZtsClientMock implements ZtsClient { } @Override + public AthenzAccessToken getAccessToken(AthenzDomain domain) { + throw new UnsupportedOperationException(); + } + + @Override + public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) { + throw new UnsupportedOperationException(); + } + + @Override public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) { throw new UnsupportedOperationException(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java deleted file mode 100644 index 41f5b65d263..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -import java.util.Objects; - -/** - * Represents a reference to a certificate and private key. - * - * @author mortent - * @author andreer - */ -public class ApplicationCertificate { - - private final String secretsKeyNamePrefix; - - public ApplicationCertificate(String secretsKeyNamePrefix) { - this.secretsKeyNamePrefix = Objects.requireNonNull(secretsKeyNamePrefix, "secretsKeyNamePrefix must be non-null"); - } - - /** The prefix of keys identifying this certificate and its private key in a key store */ - public String secretsKeyNamePrefix() { - return secretsKeyNamePrefix; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ApplicationCertificate that = (ApplicationCertificate) o; - return Objects.equals(secretsKeyNamePrefix, that.secretsKeyNamePrefix); - } - - @Override - public int hashCode() { - return Objects.hash(secretsKeyNamePrefix); - } - - @Override - public String toString() { - return "application certificate '" + secretsKeyNamePrefix + "'"; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 56a50faa4fa..0aa0df8ae2b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -1,11 +1,13 @@ package com.yahoo.vespa.hosted.controller.api.integration.certificates; +import java.util.List; import java.util.Objects; +import java.util.Optional; /** * This class is used for metadata about an application's endpoint certificate on the controller. * <p> - * It is a copy of com.yahoo.config.model.api.EndpointCertificateMetadata, but will soon be extended. + * It has more properties than com.yahoo.config.model.api.EndpointCertificateMetadata. * * @author andreer */ @@ -14,11 +16,27 @@ public class EndpointCertificateMetadata { private final String keyName; private final String certName; private final int version; + private final Optional<String> request_id; + private final Optional<List<String>> requestedDnsSans; public EndpointCertificateMetadata(String keyName, String certName, int version) { this.keyName = keyName; this.certName = certName; this.version = version; + this.request_id = Optional.empty(); + this.requestedDnsSans = Optional.empty(); + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + this.request_id = request_id; + this.requestedDnsSans = requestedDnsSans; + } + + public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { + this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans)); } public String keyName() { @@ -33,12 +51,22 @@ public class EndpointCertificateMetadata { return version; } + public Optional<String> request_id() { + return request_id; + } + + public Optional<List<String>> requestedDnsSans() { + return requestedDnsSans; + } + @Override public String toString() { return "EndpointCertificateMetadata{" + "keyName='" + keyName + '\'' + ", certName='" + certName + '\'' + ", version=" + version + + ", request_id=" + request_id + + ", requestedDnsSans=" + requestedDnsSans + '}'; } @@ -48,12 +76,14 @@ public class EndpointCertificateMetadata { if (o == null || getClass() != o.getClass()) return false; EndpointCertificateMetadata that = (EndpointCertificateMetadata) o; return version == that.version && - Objects.equals(keyName, that.keyName) && - Objects.equals(certName, that.certName); + keyName.equals(that.keyName) && + certName.equals(that.certName) && + request_id.equals(that.request_id) && + requestedDnsSans.equals(that.requestedDnsSans); } @Override public int hashCode() { - return Objects.hash(keyName, certName, version); + return Objects.hash(keyName, certName, version, request_id, requestedDnsSans); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index cc2d08c3fcd..aa0ac5f8296 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -12,7 +12,7 @@ import java.util.UUID; /** * @author tokle */ -public class ApplicationCertificateMock implements ApplicationCertificateProvider { +public class EndpointCertificateMock implements EndpointCertificateProvider { private final Map<ApplicationId, List<String>> dnsNames = new HashMap<>(); @@ -21,11 +21,12 @@ public class ApplicationCertificateMock implements ApplicationCertificateProvide } @Override - public ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { + public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) { this.dnsNames.put(applicationId, dnsNames); - return new ApplicationCertificate(String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), - applicationId.application(), - UUID.randomUUID().toString())); + String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(), + applicationId.application(), + UUID.randomUUID().toString()); + return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java index b6ad1701449..147ada51816 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java @@ -6,12 +6,12 @@ import com.yahoo.config.provision.ApplicationId; import java.util.List; /** - * Generates a certificate. + * Generates an endpoint certificate for an application instance. * * @author andreer */ -public interface ApplicationCertificateProvider { +public interface EndpointCertificateProvider { - ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); + EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 14d3e88e631..722f9c4e33b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -44,7 +44,9 @@ public interface ConfigServer { ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); - Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); + Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath); + + String getClusterControllerStatus(DeploymentId deployment, String restPath); /** * Gets the Vespa logs of the given deployment. @@ -111,15 +113,12 @@ public interface ConfigServer { TesterCloud.Status getTesterStatus(DeploymentId deployment); /** Starts tests on tester node */ - // TODO: Remove default implementation when implemented in internal repo - default String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { return "Tests started"; } + String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config); /** Gets log from tester node */ - // TODO: Remove default implementation when implemented in internal repo - default List<LogEntry> getTesterLog(DeploymentId deployment, long after) { return List.of(); } + List<LogEntry> getTesterLog(DeploymentId deployment, long after); /** Is tester node ready */ - // TODO: Remove default implementation when implemented in internal repo - default boolean isTesterReady(DeploymentId deployment) { return false; } + boolean isTesterReady(DeploymentId deployment); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index 36b13b3496a..a1c4d379b6c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -18,15 +18,13 @@ public interface TesterCloud { void startTests(URI testerUrl, Suite suite, byte[] config); /** Signals the tester to run its tests. */ - // TODO: Remove default implementation when implementations have been updated - default void startTests(DeploymentId deploymentId, Suite suite, byte[] config) {} + void startTests(DeploymentId deploymentId, Suite suite, byte[] config); /** Returns the log entries from the tester with ids after the given threshold. */ List<LogEntry> getLog(URI testerUrl, long after); /** Returns the log entries from the tester with ids after the given threshold. */ - // TODO: Remove default implementation when implementations have been updated - default List<LogEntry> getLog(DeploymentId deploymentId, long after) { return List.of(); } + List<LogEntry> getLog(DeploymentId deploymentId, long after); /** Returns the current status of the tester. */ Status getStatus(URI testerUrl); @@ -41,8 +39,7 @@ public interface TesterCloud { boolean testerReady(URI endpointUrl); /** Returns whether the test container is ready to serve */ - // TODO: Remove default implementation when implementations have been updated - default boolean testerReady(DeploymentId deploymentId) { return false; } + boolean testerReady(DeploymentId deploymentId); /** Returns whether the given URL is registered in DNS. */ boolean exists(URI endpointUrl); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 75d49e542dc..680613e8065 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -68,6 +68,11 @@ public class MemoryNameService implements NameService { @Override public List<Record> findRecords(Record.Type type, RecordData data) { + if (type == Record.Type.ALIAS && data.asString().contains("/")) { + // Validate the same expectation as of a real name service + throw new IllegalArgumentException("Finding " + Record.Type.ALIAS + " record by data should only include " + + "the FQDN name"); + } return records.stream() .filter(record -> record.type() == type && record.data().equals(data)) .collect(Collectors.toUnmodifiableList()); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java index a7dc5f81bc0..cd0e6552596 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java @@ -1,8 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.routing; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.util.List; @@ -27,17 +29,19 @@ public class RoutingGeneratorMock implements RoutingGenerator { private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>(); private final List<RoutingEndpoint> defaultEndpoints; + private final ZoneRegistry zoneRegistry; - public RoutingGeneratorMock() { - this(List.of()); - } - - public RoutingGeneratorMock(List<RoutingEndpoint> endpoints) { + public RoutingGeneratorMock(List<RoutingEndpoint> endpoints, ZoneRegistry zoneRegistry) { this.defaultEndpoints = List.copyOf(endpoints); + this.zoneRegistry = zoneRegistry; } @Override public List<RoutingEndpoint> endpoints(DeploymentId deployment) { + if (!zoneRegistry.zones().routingMethod(RoutingMethod.shared).ids().contains(deployment.zoneId())) { + throw new IllegalArgumentException(deployment.zoneId() + " does not support routing method " + + RoutingMethod.shared); + } if (routingTable.isEmpty()) return defaultEndpoints; return routingTable.getOrDefault(deployment, List.of()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java deleted file mode 100644 index 2bf2a706ee6..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(value = JsonInclude.Include.NON_NULL) -public class StatusReply { - - @JsonProperty("status") public final RotationStatus status; - @JsonProperty("lastUpdate") public final long lastUpdate; - @JsonProperty("cause") public final String cause; - @JsonProperty("agent") public final String agent; - - @JsonCreator - public StatusReply(@JsonProperty("status") RotationStatus status, - @JsonProperty("lastUpdate") long lastUpdate, - @JsonProperty("cause") String cause, - @JsonProperty("agent") String agent) { - this.status = status; - this.lastUpdate = lastUpdate; - this.cause = cause; - this.agent = agent; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java deleted file mode 100644 index aea5f6d928d..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -/** - * @author bjorncs - */ -public enum ZoneStatus { - OUT, IN -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java deleted file mode 100644 index d2e94e73b39..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class ZoneStatusReply { - - public final ZoneStatus status; - - @JsonCreator - public ZoneStatusReply(@JsonProperty("status") ZoneStatus status) { - this.status = status; - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java deleted file mode 100644 index 0b6117d516f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author bjorncs - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.routing.status; - -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index f28c877dce6..17c9e852bd9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,14 +1,11 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -20,9 +17,9 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; @@ -41,16 +38,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -58,32 +49,23 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.endpointcertificates.EndpointCertificateManager; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import java.net.URI; import java.security.Principal; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -93,7 +75,6 @@ import java.util.TreeMap; import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -119,34 +100,29 @@ public class ApplicationController { private final ArtifactRepository artifactRepository; private final ApplicationStore applicationStore; - private final RotationRepository rotationRepository; private final AccessControl accessControl; private final ConfigServer configServer; - private final RoutingGenerator routingGenerator; - private final RoutingPolicies routingPolicies; private final Clock clock; private final DeploymentTrigger deploymentTrigger; private final ApplicationPackageValidator applicationPackageValidator; private final EndpointCertificateManager endpointCertificateManager; ApplicationController(Controller controller, CuratorDb curator, - AccessControl accessControl, RotationsConfig rotationsConfig, - Clock clock, SecretStore secretStore) { + AccessControl accessControl, Clock clock, + SecretStore secretStore, FlagSource flagSource) { + this.controller = controller; this.curator = curator; this.accessControl = accessControl; this.configServer = controller.serviceRegistry().configServer(); - this.routingGenerator = controller.serviceRegistry().routingGenerator(); this.clock = clock; this.artifactRepository = controller.serviceRegistry().artifactRepository(); this.applicationStore = controller.serviceRegistry().applicationStore(); - routingPolicies = new RoutingPolicies(controller); - rotationRepository = new RotationRepository(rotationsConfig, this, curator); deploymentTrigger = new DeploymentTrigger(controller, clock); applicationPackageValidator = new ApplicationPackageValidator(controller); endpointCertificateManager = new EndpointCertificateManager(controller.zoneRegistry(), curator, secretStore, - controller.serviceRegistry().applicationCertificateProvider(), clock); + controller.serviceRegistry().endpointCertificateProvider(), clock, flagSource); // Update serialization format of all applications Once.after(Duration.ofMinutes(1), () -> { @@ -230,7 +206,7 @@ public class ApplicationController { public Map<ZoneId, List<String>> contentClustersByZone(Collection<DeploymentId> ids) { Map<ZoneId, List<String>> clusters = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (DeploymentId id : ids) - clusters.put(id.zoneId(), ImmutableList.copyOf(configServer.getContentClusters(id))); + clusters.put(id.zoneId(), List.copyOf(configServer.getContentClusters(id))); return Collections.unmodifiableMap(clusters); } @@ -248,35 +224,6 @@ public class ApplicationController { .orElse(controller.systemVersion()); } - /** Change status of all global endpoints for given deployment */ - public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - var globalEndpoints = findGlobalEndpoints(deployment); - globalEndpoints.forEach(endpoint -> { - try { - configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); - } catch (Exception e) { - throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); - } - }); - } - - /** Get global endpoint status for given deployment */ - public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { - var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>(); - findGlobalEndpoints(deployment).forEach(endpoint -> { - var status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName()); - routingEndpoints.put(endpoint, status); - }); - return Collections.unmodifiableMap(routingEndpoints); - } - - /** Find the global endpoints of given deployment */ - private List<RoutingEndpoint> findGlobalEndpoints(DeploymentId deployment) { - return routingGenerator.endpoints(deployment).stream() - .filter(RoutingEndpoint::isGlobal) - .collect(Collectors.toUnmodifiableList()); - } - /** * Creates a new application for an existing tenant. * @@ -399,7 +346,7 @@ public class ApplicationController { endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone); - endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); + endpoints = controller.routingController().registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. @@ -455,7 +402,7 @@ public class ApplicationController { for (InstanceName instance : declaredInstances) if (applicationPackage.deploymentSpec().requireInstance(instance).concerns(Environment.prod)) - application = withRotation(applicationPackage.deploymentSpec(), application, instance); + application = controller.routingController().assignRotations(application, instance); store(application); return application; @@ -501,63 +448,8 @@ public class ApplicationController { } finally { // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that // any DNS updates can be propagated as early as possible. - routingPolicies.refresh(application, applicationPackage.deploymentSpec(), zone); - } - } - - /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(DeploymentSpec deploymentSpec, LockedApplication application, InstanceName instanceName) { - try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(deploymentSpec, - application.get().require(instanceName), - rotationLock); - application = application.with(instanceName, instance -> instance.with(rotations)); - store(application); // store assigned rotation even if deployment fails - } - return application; - } - - /** - * Register endpoints for rotations assigned to given application and zone in DNS. - * - * @return the registered endpoints - */ - private Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { - var containerEndpoints = new HashSet<ContainerEndpoint>(); - boolean registerLegacyNames = deploymentSpec.instance(instance.name()) - .flatMap(DeploymentInstanceSpec::globalServiceId) - .isPresent(); - for (var assignedRotation : instance.rotations()) { - var names = new ArrayList<String>(); - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) - .scope(Endpoint.Scope.global); - - // Skip rotations which do not apply to this zone. Legacy names always point to all zones - if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { - continue; - } - - // Omit legacy DNS names when assigning rotations using <endpoints/> syntax - if (!registerLegacyNames) { - endpoints = endpoints.legacy(false); - } - - // Register names in DNS - var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); - if (rotation.isPresent()) { - endpoints.asList().forEach(endpoint -> { - controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), - RecordData.fqdn(rotation.get().name()), - Priority.normal); - names.add(endpoint.dnsName()); - }); - } - - // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) - names.add(assignedRotation.rotationId().asString()); - containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names)); + controller.routingController().policies().refresh(application, applicationPackage.deploymentSpec(), zone); } - return Collections.unmodifiableSet(containerEndpoints); } private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { @@ -611,74 +503,6 @@ public class ApplicationController { options.deployCurrentVersion); } - /** Returns the endpoints of the deployment, or empty if the request fails */ - public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) { - if ( ! getInstance(deploymentId.applicationId()) - .map(application -> application.deployments().containsKey(deploymentId.zoneId())) - .orElse(deploymentId.applicationId().instance().isTester())) - throw new NotExistsException("Deployment", deploymentId.toString()); - - try { - return findRoutingEndpoints(deploymentId).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .collect(Collectors.toUnmodifiableList()); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e); - return Collections.emptyList(); - } - } - - /** Find the routing endpoints of a given deployment. Routing endpoints are owned by the shared routing layer. */ - private List<RoutingEndpoint> findRoutingEndpoints(DeploymentId deployment) { - if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) { - return List.of(); // No shared routing layer in this zone. - } - return routingGenerator.endpoints(deployment); - } - - private Map<ClusterSpec.Id, URI> findClusterEndpoints(DeploymentId deployment) { - if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) { - return Map.of(); // No shared routing layer in this zone. - } - return routingGenerator.clusterEndpoints(deployment); - } - - /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ - public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) { - if ( ! getInstance(id.applicationId()) - .map(application -> application.deployments().containsKey(id.zoneId())) - .orElse(id.applicationId().instance().isTester())) - throw new NotExistsException("Deployment", id.toString()); - - // TODO(jvenstad): Swap to use routingPolicies first, when this is ready. - try { - var endpoints = findClusterEndpoints(id); - if ( ! endpoints.isEmpty()) - return endpoints; - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + id, e); - } - return routingPolicies.get(id).values().stream() - .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), - policy -> policy.endpointIn(controller.system()).url())); - } - - /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ - public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(Collection<DeploymentId> ids) { - Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value)); - for (DeploymentId id : ids) { - var endpoints = clusterEndpoints(id); - if ( ! endpoints.isEmpty()) { - deployments.put(id.zoneId(), endpoints); - } - } - return Collections.unmodifiableMap(deployments); - } - /** * Deletes the the given application. All known instances of the applications will be deleted. * @@ -700,7 +524,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments); for (Instance instance : application.get().instances().values()) { - removeEndpoints(instance); + controller.routingController().removeEndpointsInDns(instance); application = application.without(instance.name()); } @@ -736,24 +560,13 @@ public class ApplicationController { && application.get().deploymentSpec().instanceNames().contains(instanceId.instance())) throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first"); - removeEndpoints(application.get().require(instanceId.instance())); + controller.routingController().removeEndpointsInDns(application.get().require(instanceId.instance())); curator.writeApplication(application.without(instanceId.instance()).get()); controller.jobController().collectGarbage(); log.info("Deleted " + instanceId); }); } - private void removeEndpoints(Instance instance) { - instance.rotations().forEach(assignedRotation -> { - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); - endpoints.asList().stream() - .map(Endpoint::dnsName) - .forEach(name -> { - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); - }); - }); - } - /** * Replace any previous version of this application by this instance * @@ -829,7 +642,7 @@ public class ApplicationController { } catch (NotFoundException ignored) { // ok; already gone } finally { - routingPolicies.refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); + controller.routingController().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); } return application.with(instanceName, instance -> instance.withoutDeploymentIn(zone)); } @@ -871,15 +684,6 @@ public class ApplicationController { instance.id(), zone, platformVersion, applicationVersion, deployment.version(), deployment.applicationVersion())); } - /** Returns the rotation repository, used for managing global rotation assignments */ - public RotationRepository rotationRepository() { - return rotationRepository; - } - - public RoutingPolicies routingPolicies() { - return routingPolicies; - } - /** * Verifies that the application can be deployed to the tenant, following these rules: * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 14b5c5e02c4..3d492bc00d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -73,6 +73,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource private final NameServiceForwarder nameServiceForwarder; private final MavenRepository mavenRepository; private final Metric metric; + private final RoutingController routingController; /** * Creates a controller @@ -102,11 +103,9 @@ public class Controller extends AbstractComponent implements ApplicationIdSource metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); jobController = new JobController(this); - applicationController = new ApplicationController(this, curator, accessControl, - Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"), - clock, secretStore - ); + applicationController = new ApplicationController(this, curator, accessControl, clock, secretStore, flagSource); tenantController = new TenantController(this, curator, accessControl); + routingController = new RoutingController(this, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null")); auditLogger = new AuditLogger(curator, clock); // Record the version of this controller @@ -124,6 +123,11 @@ public class Controller extends AbstractComponent implements ApplicationIdSource /** Returns the instance controlling deployment jobs. */ public JobController jobController() { return jobController; } + /** Returns the instance controlling routing */ + public RoutingController routingController() { + return routingController; + } + /** Returns the service registry of this */ public ServiceRegistry serviceRegistry() { return serviceRegistry; @@ -147,13 +151,6 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region); } - // TODO: Model the response properly - public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, - String environment, String region, String serviceName, String restPath) { - return serviceRegistry.configServer().getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, - serviceName, restPath); - } - /** Replace the current version status by a new one */ public void updateVersionStatus(VersionStatus newStatus) { VersionStatus currentStatus = versionStatus(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java new file mode 100644 index 00000000000..74304f2e49d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -0,0 +1,244 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * The routing controller encapsulates state and methods for inspecting and manipulating DNS-level routing of traffic + * in a system. + * + * The one stop shop for all your routing needs! + * + * @author mpolden + */ +public class RoutingController { + + private static final Logger log = Logger.getLogger(RoutingController.class.getName()); + + private final Controller controller; + private final RoutingPolicies routingPolicies; + private final RotationRepository rotationRepository; + private final RoutingGenerator routingGenerator; + + public RoutingController(Controller controller, RotationsConfig rotationsConfig) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.routingPolicies = new RoutingPolicies(controller); + this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), + controller.curator()); + this.routingGenerator = controller.serviceRegistry().routingGenerator(); + } + + public RoutingPolicies policies() { + return routingPolicies; + } + + public RotationRepository rotations() { + return rotationRepository; + } + + /** Returns all legacy endpoint URLs for given deployment, including global, in the shared routing layer */ + public List<URI> legacyEndpointsOf(DeploymentId deployment) { + return routingEndpointsOf(deployment).stream() + .map(RoutingEndpoint::endpoint) + .map(URI::create) + .collect(Collectors.toUnmodifiableList()); + } + + /** Returns legacy zone endpoints for given deployment, in the shared routing layer */ + public Map<ClusterSpec.Id, URI> legacyZoneEndpointsOf(DeploymentId deployment) { + if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { + return Map.of(); + } + try { + return routingGenerator.clusterEndpoints(deployment); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + deployment, e); + return Map.of(); + } + } + + /** + * Returns all non-global endpoint URLs for given deployment, grouped by their cluster ID. If deployment supports + * {@link RoutingMethod#exclusive} endpoints defined through routing polices are returned. + */ + public Map<ClusterSpec.Id, URI> zoneEndpointsOf(DeploymentId deployment) { + if ( ! controller.applications().getInstance(deployment.applicationId()) + .map(application -> application.deployments().containsKey(deployment.zoneId())) + .orElse(deployment.applicationId().instance().isTester())) + throw new NotExistsException("Deployment", deployment.toString()); + + // In exclusively routed zones we create endpoint URLs from routing policies + if (supportsRoutingMethod(RoutingMethod.exclusive, deployment.zoneId())) { + return routingPolicies.get(deployment).values().stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), + policy -> policy.endpointIn(controller.system()) + .url())); + } + return legacyZoneEndpointsOf(deployment); + } + + /** Returns all non-global endpoint URLs for given deployments, grouped by their cluster ID and zone */ + public Map<ZoneId, Map<ClusterSpec.Id, URI>> zoneEndpointsOf(Collection<DeploymentId> deployments) { + var endpoints = new TreeMap<ZoneId, Map<ClusterSpec.Id, URI>>(Comparator.comparing(ZoneId::value)); + for (var deployment : deployments) { + var zoneEndpoints = zoneEndpointsOf(deployment); + if (!zoneEndpoints.isEmpty()) { + endpoints.put(deployment.zoneId(), zoneEndpoints); + } + } + return Collections.unmodifiableMap(endpoints); + } + + /** Change status of all global endpoints for given deployment */ + public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { + var globalEndpoints = legacyGlobalEndpointsOf(deployment); + globalEndpoints.forEach(endpoint -> { + try { + controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); + } catch (Exception e) { + throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); + } + }); + } + + /** Get global endpoint status for given deployment */ + public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { + var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>(); + legacyGlobalEndpointsOf(deployment).forEach(endpoint -> { + var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, endpoint.upstreamName()); + routingEndpoints.put(endpoint, status); + }); + return Collections.unmodifiableMap(routingEndpoints); + } + + /** Find the global endpoints of given deployment */ + private List<RoutingEndpoint> legacyGlobalEndpointsOf(DeploymentId deployment) { + return routingEndpointsOf(deployment).stream() + .filter(RoutingEndpoint::isGlobal) + .collect(Collectors.toUnmodifiableList()); + } + + /** + * Assigns one or more global rotations to given application, if eligible. The given application is implicitly + * stored, ensuring that the assigned rotation(s) are persisted when this returns. + */ + public LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { + try (RotationLock rotationLock = rotationRepository.lock()) { + var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + application.get().require(instanceName), + rotationLock); + application = application.with(instanceName, instance -> instance.with(rotations)); + controller.applications().store(application); // store assigned rotation even if deployment fails + } + return application; + } + + /** + * Register endpoints for rotations assigned to given application and zone in DNS. + * + * @return the registered endpoints + */ + public Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) { + var containerEndpoints = new HashSet<ContainerEndpoint>(); + boolean registerLegacyNames = deploymentSpec.instance(instance.name()) + .flatMap(DeploymentInstanceSpec::globalServiceId) + .isPresent(); + for (var assignedRotation : instance.rotations()) { + var names = new ArrayList<String>(); + var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); + + // Skip rotations which do not apply to this zone. Legacy names always point to all zones + if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { + continue; + } + + // Omit legacy DNS names when assigning rotations using <endpoints/> syntax + if (!registerLegacyNames) { + endpoints = endpoints.legacy(false); + } + + // Register names in DNS + var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); + if (rotation.isPresent()) { + endpoints.asList().forEach(endpoint -> { + controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), + RecordData.fqdn(rotation.get().name()), + Priority.normal); + names.add(endpoint.dnsName()); + }); + } + + // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) + names.add(assignedRotation.rotationId().asString()); + containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names)); + } + return Collections.unmodifiableSet(containerEndpoints); + } + + /** Remove endpoints in DNS for all rotations assigned to given instance */ + public void removeEndpointsInDns(Instance instance) { + instance.rotations().forEach(assignedRotation -> { + var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), + Priority.normal); + }); + }); + } + + private List<RoutingEndpoint> routingEndpointsOf(DeploymentId deployment) { + if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { + return List.of(); // No rotations/shared routing layer in this zone. + } + try { + return routingGenerator.endpoints(deployment); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoints for " + deployment, e); + return List.of(); + } + } + + private boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { + return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index a8ab847cda6..3994e05cbac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -9,7 +9,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 9a6daf026c3..c39255fd7a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -111,7 +111,7 @@ public class Endpoint { return URI.create(scheme + "://" + sanitize(namePart(name, separator)) + systemPart(system, separator) + - sanitize(instancePart(application, zone, separator)) + + sanitize(instancePart(application, separator)) + sanitize(application.application().value()) + separator + sanitize(application.tenant().value()) + @@ -144,7 +144,7 @@ public class Endpoint { return zone.region().value() + "." + zone.environment().value(); } - private static String instancePart(ApplicationId application, ZoneId zone, String separator) { + private static String instancePart(ApplicationId application, String separator) { if (application.instance().isDefault()) return ""; // Skip "default" return application.instance().value() + separator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14ca182e00e..61dc249feaa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.Notifications.When; @@ -12,15 +13,11 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -30,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.ServiceState; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -87,8 +83,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.INFO; @@ -109,11 +103,12 @@ import static java.util.stream.Collectors.toList; public class InternalStepRunner implements StepRunner { private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName()); - private static final NodeResources DEFAULT_TESTER_RESOURCES = + + static final NodeResources DEFAULT_TESTER_RESOURCES = new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any); // Must match exactly the advertised resources of an AWS instance type. Also consider that the container // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259). - private static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = + static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); static final Duration endpointTimeout = Duration.ofMinutes(15); @@ -413,17 +408,10 @@ public class InternalStepRunner implements StepRunner { logger.log(nodeList.asList().stream() .flatMap(node -> nodeDetails(node, false)) .collect(toList())); - if (nodeList.summary().converged()) { - if (endpointsAvailable(testerId, zone, logger)) { - if (testerContainersAreUp(testerId, zone, logger)) { - logger.log("Tester container successfully installed!"); - return Optional.of(running); - } - } - else if (run.stepInfo(installTester).get().startTime().get().plus(endpointTimeout).isBefore(controller.clock().instant())) { - logger.log(WARNING, "Tester failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); - return Optional.of(error); - } + + if (nodeList.summary().converged() && testerContainersAreUp(testerId, zone, logger)) { + logger.log("Tester container successfully installed!"); + return Optional.of(running); } if (run.stepInfo(installTester).get().startTime().get().plus(testerTimeout).isBefore(controller.clock().instant())) { @@ -435,16 +423,13 @@ public class InternalStepRunner implements StepRunner { } /** Returns true iff all containers in the deployment give 100 consecutive 200 OK responses on /status.html. */ - // TODO: Change implementation to only be used for real deployments when useConfigServerForTesterAPI() always returns true private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; for (URI endpoint : endpoints.get(zoneId).values()) { - boolean ready = id.instance().isTester() ? controller.jobController().cloud().testerReady(endpoint) - : controller.jobController().cloud().ready(endpoint); - + boolean ready = controller.jobController().cloud().ready(endpoint); if (!ready) { logger.log("Failed to get 100 consecutive OKs from " + endpoint); return false; @@ -456,24 +441,17 @@ public class InternalStepRunner implements StepRunner { /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - if (useConfigServerForTesterAPI(zoneId)) { - DeploymentId deploymentId = new DeploymentId(id, zoneId); - if (controller.jobController().cloud().testerReady(deploymentId)) { - return true; - } else { - logger.log("Failed to get 100 consecutive OKs from tester container for " + deploymentId); - return false; - } + DeploymentId deploymentId = new DeploymentId(id, zoneId); + if (controller.jobController().cloud().testerReady(deploymentId)) { + return true; } else { - return containersAreUp(id, zoneId, logger); + logger.log("Failed to get 100 consecutive OKs from tester container for " + deploymentId); + return false; } } - private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - if (useConfigServerForTesterAPI(zone) && id.instance().isTester()) return true; // Endpoints not used in this case, always return true - - var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zone))); + var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zone))); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; @@ -546,29 +524,16 @@ public class InternalStepRunner implements StepRunner { deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.applications().clusterEndpoints(deployments); + var endpoints = controller.routingController().zoneEndpointsOf(deployments); if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); } logEndpoints(endpoints, logger); - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if (useConfigServerForTesterAPI(zoneId)) { - if ( ! controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) { - logger.log(WARNING, "Tester container went bad!"); - return Optional.of(error); - } - } else { - if (testerEndpoint.isEmpty()) { - logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!"); - return Optional.of(error); - } - - if ( ! controller.jobController().cloud().testerReady(testerEndpoint.get())) { - logger.log(WARNING, "Tester container went bad!"); - return Optional.of(error); - } + if (!controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) { + logger.log(WARNING, "Tester container went bad!"); + return Optional.of(error); } logger.log("Starting tests ..."); @@ -578,11 +543,7 @@ public class InternalStepRunner implements StepRunner { true, endpoints, controller.applications().contentClustersByZone(deployments)); - if (useConfigServerForTesterAPI(zoneId)) { - controller.jobController().cloud().startTests(getTesterDeploymentId(id), suite, config); - } else { - controller.jobController().cloud().startTests(testerEndpoint.get(), suite, config); - } + controller.jobController().cloud().startTests(getTesterDeploymentId(id), suite, config); return Optional.of(running); } @@ -605,18 +566,7 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestLog(id); - TesterCloud.Status testStatus; - if (useConfigServerForTesterAPI(id.type().zone(controller.system()))) { - testStatus = controller.jobController().cloud().getStatus(getTesterDeploymentId(id)); - } else { - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if (testerEndpoint.isEmpty()) { - logger.log("Endpoints for tester not found -- trying again later."); - return Optional.empty(); - } - testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); - } - + TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(getTesterDeploymentId(id)); switch (testStatus) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); @@ -764,13 +714,9 @@ public class InternalStepRunner implements StepRunner { ZoneId zone = id.type().zone(controller.system()); boolean useTesterCertificate = controller.system().isPublic() && id.type().environment().isTest(); - byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(), - ! controller.system().isPublic(), + byte[] servicesXml = servicesXml(! controller.system().isPublic(), useTesterCertificate, - testerFlavorFor(id, spec) - .map(NodeResources::fromLegacyName) - .orElse(zone.region().value().contains("aws-") ? - DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES)); + testerResourcesFor(zone, spec.requireInstance(id.application().instance()))); byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version); byte[] deploymentXml = deploymentXml(id.tester(), spec.athenzDomain(), @@ -808,25 +754,18 @@ public class InternalStepRunner implements StepRunner { return new DeploymentId(runId.tester().id(), zoneId); } - private boolean useConfigServerForTesterAPI(ZoneId zoneId) { - BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); - boolean useConfigServer = useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value(); - InternalStepRunner.logger.log(LogLevel.INFO, Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.id().toString() + - " has value " + useConfigServer + " in zone " + zoneId.value()); - return useConfigServer; - } - - private static Optional<String> testerFlavorFor(RunId id, DeploymentSpec spec) { - for (DeploymentSpec.Step step : spec.steps()) - if (step.concerns(id.type().environment())) - return step.zones().get(0).testerFlavor(); - - return Optional.empty(); + static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) { + return spec.steps().stream() + .filter(step -> step.concerns(zone.environment())) + .findFirst() + .flatMap(step -> step.zones().get(0).testerFlavor()) + .map(NodeResources::fromLegacyName) + .orElse(zone.region().value().contains("aws-") ? + DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES); } /** Returns the generated services.xml content for the tester application. */ - static byte[] servicesXml(AthenzDomain domain, boolean systemUsesAthenz, boolean useTesterCertificate, - NodeResources resources) { + static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, NodeResources resources) { int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); @@ -837,7 +776,6 @@ public class InternalStepRunner implements StepRunner { "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>", resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name()); - AthenzDomain idDomain = ("vespa.vespa.cd".equals(domain.value()) ? AthenzDomain.from("vespa.vespa") : domain); String servicesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<services xmlns:deploy='vespa' version='1.0'>\n" + @@ -856,53 +794,6 @@ public class InternalStepRunner implements StepRunner { " <binding>http://*/tester/v1/*</binding>\n" + " </handler>\n" + "\n" + - " <http>\n" + - " <!-- Make sure 4080 is the first port. This will be used by the config server. -->\n" + - " <server id='default' port='4080'/>\n" + - " <server id='testertls4443' port='4443'>\n" + - " <config name=\"jdisc.http.connector\">\n" + - " <tlsClientAuthEnforcer>\n" + - " <enable>true</enable>\n" + - " <pathWhitelist>\n" + - " <item>/status.html</item>\n" + - " <item>/state/v1/config</item>\n" + - " </pathWhitelist>\n" + - " </tlsClientAuthEnforcer>\n" + - " </config>\n" + - " <ssl>\n" + - " <private-key-file>/var/lib/sia/keys/" + idDomain.value() + ".tenant.key.pem</private-key-file>\n" + - " <certificate-file>/var/lib/sia/certs/" + idDomain.value() + ".tenant.cert.pem</certificate-file>\n" + - " <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file>\n" + - " <client-authentication>want</client-authentication>\n" + - " </ssl>\n" + - " </server>\n" + - " <filtering>\n" + - (systemUsesAthenz ? - " <access-control domain='" + domain.value() + "'>\n" + // Set up dummy access control to pass validation :/ - " <exclude>\n" + - " <binding>http://*/tester/v1/*</binding>\n" + - " </exclude>\n" + - " </access-control>\n" - : "") + - " <request-chain id=\"testrunner-api\">\n" + - " <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle=\"jdisc-security-filters\">\n" + - " <config name=\"jdisc.http.filter.security.athenz.athenz-authorization-filter\">\n" + - " <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>\n" + - " <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>\n" + - " </config>\n" + - " <component id=\"com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper\" bundle=\"jdisc-security-filters\">\n" + - " <config name=\"jdisc.http.filter.security.athenz.static-request-resource-mapper\">\n" + - " <resourceName>" + domain.value() + ":tester-application</resourceName>\n" + - " <action>deploy</action>\n" + - " </config>\n" + - " </component>\n" + - " </filter>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n" + - "\n" + - " <accesslog type='json' fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S'/>\n" + - "\n" + " <nodes count=\"1\" allocated-memory=\"" + jdiscMemoryPct + "%\">\n" + " " + resourceString + "\n" + " </nodes>\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 13b3b368293..d8d9431dc22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -7,9 +7,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -183,16 +180,8 @@ public class JobController { if (step.isEmpty()) return run; - Optional<URI> testerEndpoint = testerEndpoint(id); - if (testerEndpoint.isEmpty()) - return run; - - List<LogEntry> entries; - ZoneId zone = id.type().zone(controller.system()); - if (useConfigServerForTesterAPI(zone)) - entries = cloud.getLog(new DeploymentId(id.application(), zone), run.lastTestLogEntry()); - else - entries = cloud.getLog(testerEndpoint.get(), run.lastTestLogEntry()); + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -517,7 +506,7 @@ public class JobController { } finally { // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and // tester instances have none. - controller.applications().routingPolicies().refresh(id.id(), DeploymentSpec.empty, zone); + controller.routingController().policies().refresh(id.id(), DeploymentSpec.empty, zone); } } @@ -545,14 +534,10 @@ public class JobController { .collect(toList())); } - /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ + /** Returns the tester endpoint URL, if any */ Optional<URI> testerEndpoint(RunId id) { - DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); - return controller.applications().getDeploymentEndpoints(testerId) - .stream().findAny() - .or(() -> controller.applications().routingPolicies().get(testerId).values().stream() - .findAny() - .map(policy -> policy.endpointIn(controller.system()).url())); + var testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); + return controller.routingController().zoneEndpointsOf(testerId).values().stream().findFirst(); } private void prunePackages(TenantAndApplicationId id) { @@ -597,9 +582,4 @@ public class JobController { } } - private boolean useConfigServerForTesterAPI(ZoneId zoneId) { - BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); - return useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value(); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index a6ffb56492f..5df914bad80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -40,12 +40,11 @@ public class JobMetrics { } Map<String, String> contextOf(JobId id) { - return Map.of("tenant", id.application().tenant().value(), - "application", id.application().application().value(), - "instance", id.application().instance().value(), - "job", id.type().jobName(), - "environment", id.type().environment().value(), - "region", id.type().zone(system).region().value()); + return Map.of("applicationId", id.application().toFullString(), + "tenantName", id.application().tenant().value(), + "app", id.application().application().value() + "." + id.application().instance().value(), + "test", Boolean.toString(id.type().isTest()), + "zone", id.type().zone(system).value()); } static String valueOf(RunStatus status) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index a083e2d0210..0dfe945af2b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import java.io.IOException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java index ad8c0c73754..c90d5886777 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -1,36 +1,45 @@ package com.yahoo.vespa.hosted.controller.endpointcertificates; +import com.google.common.collect.Sets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.log.LogLevel; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateMetadataSerializer; import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; /** + * Looks up stored endpoint certificate metadata, provisions new certificates if none is found, + * and refreshes certificates if a newer version is available. + * * @author andreer */ public class EndpointCertificateManager { @@ -40,64 +49,89 @@ public class EndpointCertificateManager { private final ZoneRegistry zoneRegistry; private final CuratorDb curator; private final SecretStore secretStore; - private final ApplicationCertificateProvider applicationCertificateProvider; + private final EndpointCertificateProvider endpointCertificateProvider; private final Clock clock; + private final BooleanFlag useRefreshedEndpointCertificate; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, SecretStore secretStore, - ApplicationCertificateProvider applicationCertificateProvider, - Clock clock) { + EndpointCertificateProvider endpointCertificateProvider, + Clock clock, FlagSource flagSource) { this.zoneRegistry = zoneRegistry; this.curator = curator; this.secretStore = secretStore; - this.applicationCertificateProvider = applicationCertificateProvider; + this.endpointCertificateProvider = endpointCertificateProvider; this.clock = clock; + this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource); } public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { if (!zoneRegistry.zones().directlyRouted().ids().contains(zone)) return Optional.empty(); - // Re-use certificate if already provisioned - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = + // Re-use existing certificate if already provisioned + var endpointCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id()) - .or(() -> Optional.of(provisionEndpointCertificate(instance))); + .orElseGet(() -> provisionEndpointCertificate(instance)); - // Only logs warnings for now - endpointCertificateMetadata.ifPresent(certificateMetadata -> verifyEndpointCertificate(certificateMetadata, instance, zone)); + // If feature flag set for application, look for and use refreshed certificate + if (useRefreshedEndpointCertificate.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value()) { + var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata); - return endpointCertificateMetadata; + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) { + var refreshedCertificateMetadata = new EndpointCertificateMetadata( + endpointCertificateMetadata.keyName(), + endpointCertificateMetadata.certName(), + latestAvailableVersion.getAsInt() + ); + + if (verifyEndpointCertificate(refreshedCertificateMetadata, instance, zone, "Did not refresh, problems with refreshed certificate: ")) + return Optional.of(refreshedCertificateMetadata); + } + } + + // Only log warnings + verifyEndpointCertificate(endpointCertificateMetadata, instance, zone, "Problems while verifying certificate: "); + + return Optional.of(endpointCertificateMetadata); + } + + private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { + var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); + var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); + + return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); } private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { List<ZoneId> directlyRoutedZones = zoneRegistry.zones().directlyRouted().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); - ApplicationCertificate newCertificate = applicationCertificateProvider + EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), directlyRoutedZones)); - EndpointCertificateMetadata provisionedCertificateMetadata = EndpointCertificateMetadataSerializer.fromTlsSecretsKeysString(newCertificate.secretsKeyNamePrefix()); curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); return provisionedCertificateMetadata; } - private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) { + private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone, String warningPrefix) { try { var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); - if (pemEncodedEndpointCertificate == null) return logWarning("Certificate not found in secret store"); + if (pemEncodedEndpointCertificate == null) + return logWarning(warningPrefix, "Secret store returned null for certificate"); List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate); - if (x509CertificateList.isEmpty()) return logWarning("Empty certificate list"); + if (x509CertificateList.isEmpty()) return logWarning(warningPrefix, "Empty certificate list"); if (x509CertificateList.size() < 2) - return logWarning("Only a single certificate found in chain - intermediate certificates likely missing"); + return logWarning(warningPrefix, "Only a single certificate found in chain - intermediate certificates likely missing"); Instant now = clock.instant(); Instant firstExpiry = Instant.MAX; for (X509Certificate x509Certificate : x509CertificateList) { Instant notBefore = x509Certificate.getNotBefore().toInstant(); Instant notAfter = x509Certificate.getNotAfter().toInstant(); - if (now.isBefore(notBefore)) return logWarning("Certificate is not yet valid"); - if (now.isAfter(notAfter)) return logWarning("Certificate has expired"); + if (now.isBefore(notBefore)) return logWarning(warningPrefix, "Certificate is not yet valid"); + if (now.isAfter(notAfter)) return logWarning(warningPrefix, "Certificate has expired"); if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter; } @@ -106,18 +140,21 @@ public class EndpointCertificateManager { .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME)) .map(SubjectAlternativeName::getValue).collect(Collectors.toSet()); - if (!subjectAlternativeNames.equals(Set.copyOf(dnsNamesOf(instance.id(), List.of(zone))))) - return logWarning("The list of SANs in the certificate does not match what we expect"); + if(Sets.intersection(subjectAlternativeNames, Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))).isEmpty()) { + return logWarning(warningPrefix, "No overlap between SANs in certificate and expected SANs"); + } return true; // All good then, hopefully + } catch (SecretNotFoundException s) { + return logWarning(warningPrefix, "Certificate not found in secret store"); } catch (Exception e) { log.log(LogLevel.WARNING, "Exception thrown when verifying endpoint certificate", e); return false; } } - private static boolean logWarning(String message) { - log.log(LogLevel.WARNING, message); + private static boolean logWarning(String warningPrefix, String message) { + log.log(LogLevel.WARNING, warningPrefix + message); return false; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index ccb802d314a..d88469645b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -64,8 +64,8 @@ public class MetricsReporter extends Maintainer { } private void reportRemainingRotations() { - try (RotationLock lock = controller().applications().rotationRepository().lock()) { - int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size(); + try (RotationLock lock = controller().routingController().rotations().lock()) { + int availableRotations = controller().routingController().rotations().availableRotations(lock).size(); metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index ac5612b4e3f..bdfaa9c098f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -82,7 +82,7 @@ public class RotationStatusUpdater extends Maintainer { private RotationStatus getStatus(Instance instance) { var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>(); for (var assignedRotation : instance.rotations()) { - var rotation = applications.rotationRepository().getRotation(assignedRotation.rotationId()); + var rotation = controller().routingController().rotations().getRotation(assignedRotation.rotationId()); if (rotation.isEmpty()) continue; var targets = service.getHealthStatus(rotation.get().name()).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index b730b63d426..ad411a895fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -19,7 +19,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -258,15 +258,13 @@ public class ApplicationSerializer { } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { - if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) { - object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong()); - toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); - applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); - applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); - applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); - applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); - applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); - } + applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number)); + applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField))); + applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email)); + applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString())); + applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli())); + applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); + applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -355,10 +353,8 @@ public class ApplicationSerializer { } private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) { - if (latestVersionObject.valid()) - return Optional.of(applicationVersionFromSlime(latestVersionObject)); - - return Optional.empty(); + return Optional.of(applicationVersionFromSlime(latestVersionObject)) + .filter(version -> ! version.isUnknown()); } private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) { @@ -474,22 +470,16 @@ public class ApplicationSerializer { private ApplicationVersion applicationVersionFromSlime(Inspector object) { if ( ! object.valid()) return ApplicationVersion.unknown; OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField)); - Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - if (sourceRevision.isEmpty() || applicationBuildNumber.isEmpty()) { + if (applicationBuildNumber.isEmpty()) return ApplicationVersion.unknown; - } + + Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField)); Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = Serializers.optionalString(object.field(sourceUrlField)); Optional<String> commit = Serializers.optionalString(object.field(commitField)); - if (authorEmail.isEmpty()) - return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong()); - - if (compileVersion.isEmpty() || buildTime.isEmpty()) - return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get()); - return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 2c51a9bdc00..5cd5bb59a9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -11,11 +11,10 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -84,7 +83,7 @@ public class CuratorDb { private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); - private static final Path applicationCertificateRoot = root.append("applicationCertificates"); + private static final Path endpointCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); @@ -516,11 +515,11 @@ public class CuratorDb { // -------------- Application web certificates ---------------------------- public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) { - curator.set(applicationCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata))); + curator.set(endpointCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata))); } public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) { - Optional<String> zkData = curator.getData(applicationCertificatePath(applicationId)).map(String::new); + Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new); return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString); } @@ -641,8 +640,8 @@ public class CuratorDb { return controllerRoot.append(hostname); } - private static Path applicationCertificatePath(ApplicationId id) { - return applicationCertificateRoot.append(id.serializedForm()); + private static Path endpointCertificatePath(ApplicationId id) { + return endpointCertificateRoot.append(id.serializedForm()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 702234b7634..653f224a02b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -3,9 +3,14 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + /** * (de)serializes endpoint certificate metadata * <p> @@ -26,6 +31,8 @@ public class EndpointCertificateMetadataSerializer { private final static String keyNameField = "keyName"; private final static String certNameField = "certName"; private final static String versionField = "version"; + private final static String requestIdField = "requestId"; + private final static String requestedDnsSansField = "requestedDnsSans"; public static Slime toSlime(EndpointCertificateMetadata metadata) { Slime slime = new Slime(); @@ -33,6 +40,13 @@ public class EndpointCertificateMetadataSerializer { object.setString(keyNameField, metadata.keyName()); object.setString(certNameField, metadata.certName()); object.setLong(versionField, metadata.version()); + + metadata.request_id().ifPresent(id -> object.setString(requestIdField, id)); + metadata.requestedDnsSans().ifPresent(sans -> { + Cursor cursor = object.setArray(requestedDnsSansField); + sans.forEach(cursor::addString); + }); + return slime; } @@ -44,12 +58,24 @@ public class EndpointCertificateMetadataSerializer { inspector.asString() + "-cert", 0 ); - case OBJECT: + case OBJECT: { + Optional<String> request_id = inspector.field(requestIdField).valid() ? + Optional.of(inspector.field(requestIdField).asString()) : + Optional.empty(); + + Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? + Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : + Optional.empty(); + return new EndpointCertificateMetadata( inspector.field(keyNameField).asString(), inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()) + Math.toIntExact(inspector.field(versionField).asLong()), + request_id, + requestedDnsSans ); + } default: throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); @@ -61,7 +87,7 @@ public class EndpointCertificateMetadataSerializer { } public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) { - if(zkdata.strip().startsWith("{")) { + if (zkdata.strip().startsWith("{")) { return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); } else { return fromTlsSecretsKeysString(zkdata); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index e32e8ceacca..fffe781e6e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -6,7 +6,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type; import com.yahoo.vespa.hosted.controller.deployment.Step; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 03f9bcd84e6..9e674134347 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -161,17 +161,19 @@ class RunSerializer { if ( ! versionObject.field(buildField).valid()) return ApplicationVersion.unknown; - SourceRevision revision = new SourceRevision(versionObject.field(repositoryField).asString(), - versionObject.field(branchField).asString(), - versionObject.field(commitField).asString()); long buildNumber = versionObject.field(buildField).asLong(); + // TODO jonmv: Remove source revision + Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(), + versionObject.field(branchField).asString(), + versionObject.field(commitField).asString())) + .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank()); Optional<String> authorEmail = Serializers.optionalString(versionObject.field(authorEmailField)); Optional<Version> compileVersion = Serializers.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); Optional<Instant> buildTime = Serializers.optionalInstant(versionObject.field(buildTimeField)); Optional<String> sourceUrl = Serializers.optionalString(versionObject.field(sourceUrlField)); Optional<String> commit = Serializers.optionalString(versionObject.field(commitField)); - return new ApplicationVersion(Optional.of(revision), OptionalLong.of(buildNumber), authorEmail, + return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit); } @@ -244,12 +246,11 @@ class RunSerializer { private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { versionsObject.setString(platformVersionField, platformVersion.toString()); - if ( ! applicationVersion.isUnknown()) { - versionsObject.setString(repositoryField, applicationVersion.source().get().repository()); - versionsObject.setString(branchField, applicationVersion.source().get().branch()); - versionsObject.setString(commitField, applicationVersion.source().get().commit()); - versionsObject.setLong(buildField, applicationVersion.buildNumber().getAsLong()); - } + applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number)); + // TODO jonmv: Remove source revision. + applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository)); + applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch)); + applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit)); applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email)); applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString())); applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java index 841cb387e54..e5adccc850c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.time.Instant; import java.util.Optional; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java index 789f6393683..cfd92e8a7f4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.HashSet; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 35128466e4d..7f938885cac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -10,7 +10,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 00e0bcd36ad..c06194fcd73 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -30,7 +30,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -840,7 +840,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .forEach(globalEndpointUrls::add); // Per-cluster endpoints. These are backed by load balancers. - var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); + var routingPolicies = controller.routingController().policies().get(instance.id()).values(); for (var policy : routingPolicies) { policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) @@ -848,6 +848,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .forEach(globalEndpointUrls::add); } + // TODO(mpolden): Remove once clients stop expecting this field var globalRotationsArray = object.setArray("globalRotations"); globalEndpointUrls.forEach(globalRotationsArray::addString); @@ -1027,22 +1028,60 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); - // Add endpoint(s) defined by routing policies + // Add zone endpoints defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) { + for (var policy : controller.routingController().policies().get(deploymentId).values()) { if (!policy.status().isActive()) continue; - Cursor endpointObject = endpointArray.addObject(); - Endpoint endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.id().cluster().value()); - endpointObject.setBool("tls", endpoint.tls()); - endpointObject.setString("url", endpoint.url().toString()); + { + var endpointObject = endpointArray.addObject(); + var endpoint = policy.endpointIn(controller.system()); + endpointObject.setString("cluster", policy.id().cluster().value()); + endpointObject.setBool("tls", endpoint.tls()); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + } + // Add global endpoints that point to this policy + for (var endpoint : policy.globalEndpointsIn(controller.system()).asList()) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", policy.id().cluster().value()); + endpointObject.setBool("tls", endpoint.tls()); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + } + } + // Add zone endpoints served by shared routing layer + for (var clusterAndUrl : controller.routingController().legacyZoneEndpointsOf(deploymentId).entrySet()) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", clusterAndUrl.getKey().value()); + endpointObject.setBool("tls", true); + endpointObject.setString("url", clusterAndUrl.getValue().toString()); + endpointObject.setString("scope", endpointScopeString(Endpoint.Scope.zone)); + } + // Add global endpoints served by shared routing layer + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + var instance = application.instances().get(deploymentId.applicationId().instance()); + if (deploymentId.zoneId().environment().isProduction()) { // Global endpoints can only point to production deployments + for (var rotation : instance.rotations()) { + var endpoints = instance.endpointsIn(controller.system(), rotation.endpointId()) + .legacy(false) + .scope(Endpoint.Scope.global) + .asList(); + for (var endpoint : endpoints) { + var endpointObject = endpointArray.addObject(); + endpointObject.setString("cluster", rotation.clusterId().value()); + endpointObject.setBool("tls", true); + endpointObject.setString("url", endpoint.url().toString()); + endpointObject.setString("scope", endpointScopeString(endpoint.scope())); + } + } } - // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may - // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to - // ask the routing layer here + // serviceUrls contains all valid endpoints for this deployment, including global. The name of these endpoints + // may contain the cluster name (if non-default). Since the controller has no knowledge of clusters for legacy + // endpoints, we can't generate these URLs on-the-fly and we have to query the routing layer. + // TODO(mpolden): Remove this once all clients stop reading this. Cursor serviceUrlArray = response.setArray("serviceUrls"); - controller.applications().getDeploymentEndpoints(deploymentId) + controller.routingController().legacyEndpointsOf(deploymentId) .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); @@ -1053,12 +1092,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId()) .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli())); - Application application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); DeploymentStatus status = controller.jobController().deploymentStatus(application); application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); sourceRevisionToSlime(deployment.applicationVersion().source(), response); - Instance instance = application.instances().get(deploymentId.applicationId().instance()); if (instance != null) { if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) toSlime(instance.rotations(), instance.rotationStatus(), deployment, response); @@ -1186,7 +1223,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent); + controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); } /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ @@ -1197,7 +1234,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { long timestamp = controller.clock().instant().getEpochSecond(); var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); - controller.applications().setGlobalRotationStatus(deployment, endpointStatus); + controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); } private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { @@ -1205,7 +1242,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ZoneId.from(environment, region)); Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); - controller.applications().globalRotationStatus(deploymentId) + controller.routingController().globalRotationStatus(deploymentId) .forEach((endpoint, status) -> { array.addString(endpoint.upstreamName()); Cursor statusObject = array.addObject(); @@ -1331,10 +1368,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { - Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), - new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), - controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)), + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + + if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) { + String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, restPath); + return new HtmlResponse(result); + } + + Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath); + ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(), + deploymentId.applicationId(), + controller.zoneRegistry().getConfigServerApiUris(deploymentId.zoneId()), request.getUri()); response.setResponse(result, serviceName, restPath); return response; @@ -1674,7 +1718,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.applications().clusterEndpoints(deployments), + controller.routingController().zoneEndpointsOf(deployments), controller.applications().contentClustersByZone(deployments))); } @@ -2063,6 +2107,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return "UNKNOWN"; } + private static String endpointScopeString(Endpoint.Scope scope) { + switch (scope) { + case global: return "global"; + case zone: return "zone"; + } + throw new IllegalArgumentException("Unknown endpoint scope " + scope); + } + private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) .filter(cls::isInstance) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java new file mode 100644 index 00000000000..99884875a64 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java @@ -0,0 +1,30 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +/** + * @author freva + */ +public class HtmlResponse extends HttpResponse { + + private final String content; + + public HtmlResponse(String content) { + super(200); + this.content = content; + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(content.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { return "text/html"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 9ea530c7886..73cdf28c366 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -415,9 +415,11 @@ class JobControllerApiHandlerHelper { versionObject.setLong("build", version.buildNumber().getAsLong()); Cursor sourceObject = versionObject.setObject("source"); - sourceObject.setString("gitRepository", version.source().get().repository()); - sourceObject.setString("gitBranch", version.source().get().branch()); - sourceObject.setString("gitCommit", version.source().get().commit()); + version.source().ifPresent(source -> { + sourceObject.setString("gitRepository", source.repository()); + sourceObject.setString("gitBranch", source.branch()); + sourceObject.setString("gitCommit", source.commit()); + }); version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url)); version.commit().ifPresent(commit -> versionObject.setString("commit", commit)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 2cadd864df1..ad847621c3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.restapi.Path; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index c168a057bfb..7d8c5922c3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -13,7 +13,7 @@ import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index be459614880..1815628a1ee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -79,7 +80,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var zone = zoneFrom(path); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.applications().routingPolicies().setGlobalRoutingStatus(zone, status); + controller.routingController().policies().setGlobalRoutingStatus(zone, status); } else { controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); } @@ -92,14 +93,14 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var slime = new Slime(); var root = slime.setObject(); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { - var zonePolicy = controller.applications().routingPolicies().get(zone); - zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingType.policy); + var zonePolicy = controller.routingController().policies().get(zone); + zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); } else { // Rotation status per zone only exposes in/out status, no agent or time of change. var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out, GlobalRouting.Agent.operator, Instant.EPOCH); - zoneStatusToSlime(root, zone, globalRouting, RoutingType.rotation); + zoneStatusToSlime(root, zone, globalRouting, RoutingMethod.shared); } return new SlimeJsonResponse(slime); } @@ -115,11 +116,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, "", agent.name(), controller.clock().instant().getEpochSecond()); - controller.applications().setGlobalRotationStatus(deployment, endpointStatus); + controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); } // Set policy status - controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent); + controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } @@ -131,7 +132,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { // Include status from rotation if (rotationCanRouteTo(deployment.zoneId(), instance)) { - var rotationStatus = controller.applications().globalRotationStatus(deployment); + var rotationStatus = controller.routingController().globalRotationStatus(deployment); // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. var endpointStatus = rotationStatus.values().stream().findFirst(); if (endpointStatus.isPresent()) { @@ -147,12 +148,12 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { : GlobalRouting.Status.out; deploymentStatusToSlime(deploymentsObject.addObject(), deployment, new GlobalRouting(status, agent, changedAt), - RoutingType.rotation); + RoutingMethod.shared); } } // Include status from routing policies - var routingPolicies = controller.applications().routingPolicies().get(deployment); + var routingPolicies = controller.routingController().policies().get(deployment); for (var policy : routingPolicies.values()) { deploymentStatusToSlime(deploymentsObject.addObject(), policy); } @@ -165,8 +166,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return !instance.rotations().isEmpty() && instance.deployments().containsKey(zone); } - private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingType routingType) { - object.setString("routingType", routingType.name()); + private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) { + object.setString("routingMethod", asString(method)); object.setString("environment", zone.environment().value()); object.setString("region", zone.region().value()); object.setString("status", asString(globalRouting.status())); @@ -174,8 +175,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); } - private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingType routingType) { - object.setString("routingType", routingType.name()); + private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingMethod method) { + object.setString("routingMethod", asString(method)); object.setString("instance", deployment.applicationId().serializedForm()); object.setString("environment", deployment.zoneId().environment().value()); object.setString("region", deployment.zoneId().region().value()); @@ -186,7 +187,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { private static void deploymentStatusToSlime(Cursor object, RoutingPolicy policy) { deploymentStatusToSlime(object, new DeploymentId(policy.id().owner(), policy.id().zone()), - policy.status().globalRouting(), RoutingType.policy); + policy.status().globalRouting(), RoutingMethod.exclusive); } private DeploymentId deploymentFrom(Path path) { @@ -219,12 +220,12 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } } - private enum RoutingType { - /** Global routing is configured by use of an {@link com.yahoo.vespa.hosted.controller.application.AssignedRotation} */ - rotation, - - /** Global routing is configured by a {@link com.yahoo.vespa.hosted.controller.routing.RoutingPolicy} */ - policy, + private static String asString(RoutingMethod method) { + switch (method) { + case shared: return "shared"; + case exclusive: return "exclusive"; + default: return "unknonwn"; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index ca989d3323e..d9ffe8a251c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -17,7 +17,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeStream; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 961a3674471..cb45c982a0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -91,6 +91,13 @@ public class RoutingPolicies { } } + /** Returns whether global DNS records should be updated for given application and zone */ + private boolean updateGlobalDnsOf(ApplicationId application, ZoneId zone) { + if (application.instance().isTester()) return false; + if (!zone.environment().isProduction()) return false; + return true; + } + /** Set the status of all global endpoints in given zone */ public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { try (var lock = db.lockRoutingPolicies()) { @@ -146,7 +153,9 @@ public class RoutingPolicies { routeEntry.getKey().endpointId(), controller.system()); controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); } - staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal)); + staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, + RecordData.fqdn(t.name().value()), + Priority.normal)); } } @@ -262,7 +271,7 @@ public class RoutingPolicies { /** Compute all endpoint IDs for given load balancer */ private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints + if (!zone.environment().isProduction()) { // Only production deployments have configurable endpoints return Set.of(); } var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 83f80488ee5..50e567b2024 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; @@ -178,7 +178,7 @@ public class ControllerTest { public void testGlobalRotations() { // Setup ControllerTester tester = this.tester.controllerTester(); - ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()); + ZoneId zone = ZoneId.from("prod", "us-west-1"); ApplicationId app = ApplicationId.from("tenant", "app1", "default"); DeploymentId deployment = new DeploymentId(app, zone); tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( @@ -190,7 +190,7 @@ public class ControllerTest { new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1") )); - Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().applications().globalRotationStatus(deployment); + Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().routingController().globalRotationStatus(deployment); Supplier<List<EndpointStatus>> upstreamOneEndpoints = () -> { return globalRotationStatus.get() .entrySet().stream() @@ -206,7 +206,7 @@ public class ControllerTest { // Set the global rotations out of service EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond()); - tester.controller().applications().setGlobalRotationStatus(deployment, status); + tester.controller().routingController().setGlobalRotationStatus(deployment, status); assertEquals(2, upstreamOneEndpoints.get().size()); assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test"))); @@ -517,9 +517,9 @@ public class ControllerTest { context.submit(applicationPackage); tester.applications().deleteApplication(context.application().id(), tester.controllerTester().credentialsFor(context.application().id().tenant())); - try (RotationLock lock = tester.applications().rotationRepository().lock()) { + try (RotationLock lock = tester.controller().routingController().rotations().lock()) { assertTrue("Rotation is unassigned", - tester.applications().rotationRepository().availableRotations(lock) + tester.controller().routingController().rotations().availableRotations(lock) .containsKey(new RotationId("rotation-id-01"))); } context.flushDnsUpdates(); @@ -705,7 +705,7 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var prodZone = ZoneId.from("prod", "us-west-1"); - tester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(prodZone)); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); var applicationPackage = new ApplicationPackageBuilder().environment(prodZone.environment()) .region(prodZone.region()) .build(); @@ -722,7 +722,7 @@ public class ControllerTest { (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + ".vespa.oath.cloud"))) .collect(Collectors.toUnmodifiableList()), - tester.controllerTester().serviceRegistry().applicationCertificateMock().dnsNamesOf(context1.instanceId())); + tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId())); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); @@ -732,8 +732,7 @@ public class ControllerTest { var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); var devZone = ZoneId.from("dev", "us-east-1"); - // Deploy app2, after "removing" direct routing everywhere - tester.controllerTester().zoneRegistry().setDirectlyRouted(); + // Deploy app2 in a zone with shared routing tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); assertTrue("Application deployed and activated", tester.configServer().application(context2.instanceId(), devZone).get().activated()); @@ -774,4 +773,19 @@ public class ControllerTest { } } + @Test + public void testDeployWithoutSourceRevision() { + var context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + // Submit without source revision + context.submit(applicationPackage, Optional.empty()) + .deploy(); + assertEquals("Deployed application", 1, context.instance().deployments().size()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 5318d6bdefd..a816f82c044 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -97,6 +97,10 @@ public final class ControllerTester { new ServiceRegistryMock()); } + public ControllerTester(ServiceRegistryMock serviceRegistryMock) { + this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), serviceRegistryMock); + } + public ControllerTester(RotationsConfig rotationsConfig) { this(rotationsConfig, new MockCuratorDb()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index c90abe5ed5c..3e1f468691c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -33,7 +34,9 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; @@ -44,6 +47,7 @@ import java.math.BigInteger; import java.net.URI; import java.security.KeyPair; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.HashSet; @@ -211,12 +215,20 @@ public class DeploymentContext { /** Flush all pending DNS updates */ public DeploymentContext flushDnsUpdates() { - tester.nameServiceDispatcher().run(); + flushDnsUpdates(Integer.MAX_VALUE); assertTrue("All name service requests dispatched", tester.controller().curator().readNameServiceQueue().requests().isEmpty()); return this; } + /** Flush count pending DNS updates */ + public DeploymentContext flushDnsUpdates(int count) { + var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator()), count); + dispatcher.run(); + return this; + } + /** Add a routing policy for this in given zone, with status set to active */ public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { return addRoutingPolicy(instanceId, zone, active); @@ -241,16 +253,16 @@ public class DeploymentContext { /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { - return submit(applicationPackage, defaultSourceRevision); + return submit(applicationPackage, Optional.of(defaultSourceRevision)); } /** Submit given application package for deployment */ - public DeploymentContext submit(ApplicationPackage applicationPackage, SourceRevision sourceRevision) { + public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { var projectId = tester.controller().applications() .requireApplication(applicationId) .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. - lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(), + lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), Optional.empty(), projectId, applicationPackage, new byte[0]); return this; } @@ -397,11 +409,6 @@ public class DeploymentContext { return this; } - /** Sets a single endpoint in the routing layer for the instance in this */ - public DeploymentContext setEndpoints(ZoneId zone) { - return setEndpoints(zone, false); - } - /** Deploy default application package, start a run for that change and return its ID */ public RunId newRun(JobType type) { submit(); @@ -427,7 +434,6 @@ public class DeploymentContext { configServer().convergeServices(instanceId, JobType.systemTest.zone(tester.controller().system())); configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system())); setEndpoints(JobType.systemTest.zone(tester.controller().system())); - setTesterEndpoints(JobType.systemTest.zone(tester.controller().system())); runner.run(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent()); @@ -452,21 +458,12 @@ public class DeploymentContext { // Provision load balancers in directly routed zones, unless explicitly deferred if (provisionLoadBalancerIn(zone)) { - if (job.type().isTest()) { - var testerDeployment = new DeploymentId(testerId.id(), zone); - configServer().putLoadBalancers(zone, List.of(new LoadBalancer(testerDeployment.toString(), - testerDeployment.applicationId(), - ClusterSpec.Id.from("default"), - HostName.from("lb-host"), - LoadBalancer.State.active, - Optional.of("dns-zone")))); - } configServer().putLoadBalancers(zone, List.of(new LoadBalancer(deployment.toString(), deployment.applicationId(), ClusterSpec.Id.from("default"), - HostName.from("lb-host"), + HostName.from("lb-0--" + instanceId.serializedForm() + "--" + zone.toString()), LoadBalancer.State.active, - Optional.of("dns-zone")))); + Optional.of("dns-zone-1")))); } // First step is always a deployment. @@ -517,18 +514,10 @@ public class DeploymentContext { return run; } - /** Sets a single endpoint in the routing layer for the tester instance in this */ - private DeploymentContext setTesterEndpoints(ZoneId zone) { - return setEndpoints(zone, true); - } - - /** Sets a single endpoint in the routing layer; this matches that required for the tester */ - private DeploymentContext setEndpoints(ZoneId zone, boolean tester) { - if (isDirectlyRouted(zone)) return this; + /** Sets a single endpoint in the routing layer */ + DeploymentContext setEndpoints(ZoneId zone) { + if (!supportsRoutingMethod(RoutingMethod.shared, zone)) return this; var id = instanceId; - if (tester) { - id = testerId.id(); - } routing.putEndpoints(new DeploymentId(id, zone), Collections.singletonList(new RoutingEndpoint(String.format("https://%s--%s--%s.%s.%s.vespa:43", id.instance().value(), @@ -574,12 +563,7 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); runner.advance(currentRun(job)); - if (provisionLoadBalancerIn(zone)) { // Endpoints are available immediately after deployment in directly routed zones - assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); - } else { - assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); - setTesterEndpoints(zone); - } + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); runner.advance(currentRun(job)); } @@ -615,11 +599,12 @@ public class DeploymentContext { /** Returns whether a load balancer is expected to be provisioned in given zone */ private boolean provisionLoadBalancerIn(ZoneId zone) { - return !deferLoadBalancerProvisioning.contains(zone.environment()) && isDirectlyRouted(zone); + return !deferLoadBalancerProvisioning.contains(zone.environment()) && + supportsRoutingMethod(RoutingMethod.exclusive, zone); } - private boolean isDirectlyRouted(ZoneId zone) { - return tester.controller().zoneRegistry().zones().directlyRouted().ids().contains(zone); + private boolean supportsRoutingMethod(RoutingMethod method, ZoneId zone) { + return tester.controller().zoneRegistry().zones().routingMethod(method).ids().contains(zone); } private JobId jobId(JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5b5c6b61357..938b801b88d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -20,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest; -import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.controller.maintenance.OutstandingChangeDeployer; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; @@ -48,7 +46,6 @@ public class DeploymentTester { public static final TenantAndApplicationId appId = TenantAndApplicationId.from("tenant", "application"); public static final ApplicationId instanceId = appId.defaultInstance(); - public static final TesterId testerId = TesterId.of(instanceId); private final ControllerTester tester; private final JobController jobs; @@ -58,7 +55,6 @@ public class DeploymentTester { private final Upgrader upgrader; private final ReadyJobsTrigger readyJobsTrigger; private final OutstandingChangeDeployer outstandingChangeDeployer; - private final NameServiceDispatcher nameServiceDispatcher; public JobController jobs() { return jobs; } public RoutingGeneratorMock routing() { return routing; } @@ -92,8 +88,6 @@ public class DeploymentTester { upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); - nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), maintenanceInterval, jobControl, - Integer.MAX_VALUE); routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock. // Get deployment job logs to stderr. @@ -112,10 +106,6 @@ public class DeploymentTester { public OutstandingChangeDeployer outstandingChangeDeployer() { return outstandingChangeDeployer; } - public NameServiceDispatcher nameServiceDispatcher() { - return nameServiceDispatcher; - } - public DeploymentTester atMondayMorning() { return at(tester.clock().instant().atZone(ZoneOffset.UTC) .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 112b8066847..db07aff34e5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -4,14 +4,14 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; @@ -186,7 +186,6 @@ public class InternalStepRunnerTest { tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); } @Test @@ -258,6 +257,9 @@ public class InternalStepRunnerTest { @Test public void alternativeEndpointsAreDetected() { + var systemTestZone = JobType.systemTest.zone(system()); + var stagingZone = JobType.stagingTest.zone(system()); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); app.newRun(JobType.systemTest); tester.runner().run();; tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); @@ -312,8 +314,6 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); - assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(app.testerId().id(), JobType.systemTest.zone(system()))).get(0).endpoint()), - tester.cloud().testerUrl()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString()); @@ -423,9 +423,12 @@ public class InternalStepRunnerTest { @Test public void certificateTimeoutAbortsJob() { tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd); - tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); + var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); RunId id = app.startSystemTestTests(); @@ -449,11 +452,45 @@ public class InternalStepRunnerTest { "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; @Test + public void generates_correct_tester_flavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <instance id='first'>\n" + + " <test tester-flavor=\"d-6-16-100\" />\n" + + " <prod>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='second'>\n" + + " <test />\n" + + " <staging />\n" + + " <prod tester-flavor=\"d-6-16-100\">\n" + + " <parallel>\n" + + " <region active=\"true\">us-east-3</region>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first")); + assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources); + + NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second")); + assertEquals(6, secondResources.vcpu(), 1e-9); + assertEquals(16, secondResources.memoryGb(), 1e-9); + assertEquals(100, secondResources.diskGb(), 1e-9); + } + + @Test public void generates_correct_services_xml_test() { - assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"), - true, - false, - new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); + assertFile("test_runner_services.xml-cd", + new String(InternalStepRunner.servicesXml( + true, + false, + new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); } private void assertFile(String resourceName, String actualContent) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index ce22e0772ff..f31038a7aba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java index 808d9ca05c4..3f8e91dec58 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java @@ -3,18 +3,31 @@ package com.yahoo.vespa.hosted.controller.endpointcertificates; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; import org.junit.Test; +import javax.security.auth.x500.X500Principal; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Clock; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Optional; -import java.util.Set; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -22,19 +35,74 @@ import static org.junit.Assert.assertTrue; */ public class EndpointCertificateManagerTest { + private final SecretStoreMock secretStore = new SecretStoreMock(); + private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main); + private final MockCuratorDb mockCuratorDb = new MockCuratorDb(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); + private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource(); + private final Clock clock = Clock.systemUTC(); + private final EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); + + private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); + private static final X509Certificate testCertificate = X509CertificateBuilder + .fromKeypair( + testKeyPair, + new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES), + SignatureAlgorithm.SHA256_WITH_ECDSA, + X509CertificateBuilder.generateRandomSerialNumber()) + .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud") + .build(); + + private final Instance testInstance = new Instance(ApplicationId.defaultId()); + private final String testKeyName = "testKeyName"; + private final String testCertName = "testCertName"; + private ZoneId testZone; + + @Before + public void setUp() { + zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones()); + testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId(); + } + + @Test + public void provisions_new_certificate() { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, endpointCertificateMetadata.get().version()); + } + @Test - public void getEndpointCertificate() { - SecretStoreMock secretStore = new SecretStoreMock(); - ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main); - zoneRegistryMock.setDirectlyRouted(Set.copyOf(zoneRegistryMock.zones().all().zones())); - MockCuratorDb mockCuratorDb = new MockCuratorDb(); - ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock(); - Clock clock = Clock.systemUTC(); - EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, applicationCertificateMock, clock); - ZoneId id = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId(); - Instance instance = new Instance(ApplicationId.defaultId()); - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, id); + public void reuses_stored_certificate_metadata() { + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(7, endpointCertificateMetadata.get().version()); + } + + @Test + public void uses_refreshed_certificate_when_available_and_valid() { + inMemoryFlagSource.withBooleanFlag(Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.id(), true); + + secretStore.setSecret(testKeyName, "secret-key", 7); + secretStore.setSecret(testCertName, "cert", 7); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(8, endpointCertificateMetadata.get().version()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index eb8af62a0e6..5d93881129f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -282,6 +283,21 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return TesterCloud.Status.SUCCESS; } + @Override + public String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { + return "Tests started"; + } + + @Override + public List<LogEntry> getTesterLog(DeploymentId deployment, long after) { + return List.of(); + } + + @Override + public boolean isTesterReady(DeploymentId deployment) { + return false; + } + /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>()); @@ -404,8 +420,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer // Returns a canned example response @Override - public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, - String environment, String region, String serviceName, String restPath) { + public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) { Map<String,List<?>> root = new HashMap<>(); List<Map<?,?>> resources = new ArrayList<>(); Map<String,String> resource = new HashMap<>(); @@ -416,6 +431,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override + public String getClusterControllerStatus(DeploymentId deployment, String restPath) { + return "<h1>OK</h1>"; + } + + @Override public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) { endpoints.put(upstreamName, status); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java index a14b7b82d67..c088965c2ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -45,4 +46,8 @@ public class SecretStoreMock extends AbstractComponent implements SecretStore { return secrets.getOrDefault(key, new TreeMap<>()).get(version); } + @Override + public List<Integer> listSecretVersions(String key) { + return List.copyOf(secrets.getOrDefault(key, new TreeMap<>()).keySet()); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index bd82807342e..323b86be1d3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -42,9 +42,9 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); - private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS); + private final RoutingGeneratorMock routingGeneratorMock; private final MockMailer mockMailer = new MockMailer(); - private final ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); private final MockContactRetriever mockContactRetriever = new MockContactRetriever(); private final MockIssueHandler mockIssueHandler = new MockIssueHandler(); @@ -64,6 +64,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); this.configServerMock = new ConfigServerMock(zoneRegistryMock); + this.routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS, zoneRegistryMock); } @Inject @@ -101,8 +102,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public ApplicationCertificateMock applicationCertificateProvider() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateProvider() { + return endpointCertificateMock; } @Override @@ -212,8 +213,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return artifactRepositoryMock; } - public ApplicationCertificateMock applicationCertificateMock() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateMock() { + return endpointCertificateMock; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 4cd50625ca3..764c2003c46 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -1,9 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; @@ -11,6 +12,7 @@ import com.yahoo.config.provision.zone.ZoneList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -23,22 +25,22 @@ import java.util.stream.Collectors; public class ZoneFilterMock implements ZoneList { private final List<ZoneApi> zones; - private final Set<ZoneApi> directlyRouted; + private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods; private final boolean negate; - private ZoneFilterMock(List<ZoneApi> zones, Set<ZoneApi> directlyRouted, boolean negate) { + private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods, boolean negate) { this.zones = zones; - this.directlyRouted = directlyRouted; + this.zoneRoutingMethods = zoneRoutingMethods; this.negate = negate; } - public static ZoneFilter from(Collection<ZoneApi> zones, Set<ZoneApi> directlyRouted) { - return new ZoneFilterMock(List.copyOf(zones), Set.copyOf(directlyRouted), false); + public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> routingMethods) { + return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false); } @Override public ZoneList not() { - return new ZoneFilterMock(zones, directlyRouted, ! negate); + return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate); } @Override @@ -53,7 +55,12 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList directlyRouted() { - return filter(directlyRouted::contains); + return routingMethod(RoutingMethod.exclusive); + } + + @Override + public ZoneList routingMethod(RoutingMethod method) { + return filter(zone -> zoneRoutingMethods.getOrDefault(zone, Set.of()).contains(method)); } @Override @@ -93,7 +100,7 @@ public class ZoneFilterMock implements ZoneList { condition.negate().test(zone) : condition.test(zone)) .collect(Collectors.toList()), - directlyRouted, false); + zoneRoutingMethods, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 3cb7a13dd54..a5e054d6068 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; @@ -34,11 +35,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); - private List<ZoneApi> zones = List.of(); + private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); + private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + + private List<? extends ZoneApi> zones; private SystemName system; private UpgradePolicy upgradePolicy = null; - private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); - private Set<ZoneApi> directlyRouted = Set.of(); /** * This sets the default list of zones contained in this. If your test need a particular set of zones, use @@ -46,20 +48,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry */ public ZoneRegistryMock(SystemName system) { this.system = system; - setZones(List.of( - ZoneApiMock.fromId("test.us-east-1"), - ZoneApiMock.fromId("staging.us-east-3"), - ZoneApiMock.fromId("dev.us-east-1"), - ZoneApiMock.fromId("dev.aws-us-east-2a"), - ZoneApiMock.fromId("perf.us-east-3"), - ZoneApiMock.fromId("prod.aws-us-east-1a"), - ZoneApiMock.fromId("prod.ap-northeast-1"), - ZoneApiMock.fromId("prod.ap-northeast-2"), - ZoneApiMock.fromId("prod.ap-southeast-1"), - ZoneApiMock.fromId("prod.us-east-3"), - ZoneApiMock.fromId("prod.us-west-1"), - ZoneApiMock.fromId("prod.us-central-1"), - ZoneApiMock.fromId("prod.eu-west-1"))); + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), + ZoneApiMock.fromId("dev.us-east-1"), + ZoneApiMock.fromId("dev.aws-us-east-2a"), + ZoneApiMock.fromId("perf.us-east-3"), + ZoneApiMock.fromId("prod.aws-us-east-1a"), + ZoneApiMock.fromId("prod.ap-northeast-1"), + ZoneApiMock.fromId("prod.ap-northeast-2"), + ZoneApiMock.fromId("prod.ap-southeast-1"), + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1")); + // All zones use a shared routing method by default + setRoutingMethod(this.zones, RoutingMethod.shared); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -72,7 +75,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setZones(List<ZoneApi> zones) { + public ZoneRegistryMock setZones(List<? extends ZoneApi> zones) { this.zones = zones; return this; } @@ -96,12 +99,25 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setDirectlyRouted(ZoneApi... zones) { - return setDirectlyRouted(Set.of(zones)); + public ZoneRegistryMock exclusiveRoutingIn(ZoneApi... zones) { + return exclusiveRoutingIn(List.of(zones)); + } + + public ZoneRegistryMock exclusiveRoutingIn(List<? extends ZoneApi> zones) { + return setRoutingMethod(zones, RoutingMethod.exclusive); + } + + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) { + return setRoutingMethod(zone, Set.of(routingMethods)); + } + + public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) { + zones.forEach(zone -> setRoutingMethod(zone, Set.of(routingMethods))); + return this; } - public ZoneRegistryMock setDirectlyRouted(Set<ZoneApi> zones) { - directlyRouted = zones; + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, Set<RoutingMethod> routingMethods) { + this.zoneRoutingMethods.put(zone, Set.copyOf(routingMethods)); return this; } @@ -112,7 +128,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public ZoneFilter zones() { - return ZoneFilterMock.from(zones, directlyRouted); + return ZoneFilterMock.from(zones, zoneRoutingMethods); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 9de0020ce4a..b3a6ef53d2b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -375,12 +375,11 @@ public class JobRunnerTest { jobs.finish(jobs.last(id, systemTest).get().id()); } - Map<String, String> context = Map.of("tenant", "tenant", - "application", "real", - "instance", "default", - "job", "system-test", - "environment", "test", - "region", "us-east-1"); + Map<String, String> context = Map.of("applicationId", "tenant.real.default", + "tenantName", "tenant", + "app", "real.default", + "test", "true", + "zone", "test.us-east-1"); MetricsMock metric = ((MetricsMock) tester.controller().metric()); assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 442a2fd1853..ceebbb8254f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -17,6 +17,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -47,7 +48,7 @@ public class OutstandingChangeDeployerTest { assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong()); - app1.submit(applicationPackage, new SourceRevision("repository1", "master", "cafed00d")); + app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d"))); assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index c4cb01f8164..dde1333eb5f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java new file mode 100644 index 00000000000..7428b9901a2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +public class EndpointCertificateMetadataSerializerTest { + + private EndpointCertificateMetadata sample = + new EndpointCertificateMetadata("keyName", "certName", 1); + private EndpointCertificateMetadata sampleWithRequestMetadata = + new EndpointCertificateMetadata("keyName", "certName", 1, "requestId", List.of("SAN1", "SAN2")); + + @Test + public void serialize() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}", + EndpointCertificateMetadataSerializer.toSlime(sample).toString()); + } + + @Test + public void serializeWithRequestMetadata() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}", + EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); + } + + @Test + public void deserializeFromString() { + assertEquals( + new EndpointCertificateMetadata("foo-key", "foo-cert", 0), + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString("foo")); + } + + @Test + public void deserializeFromJson() { + assertEquals( + sample, + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); + } + + @Test + public void deserializeFromJsonWithRequestMetadata() { + assertEquals( + sampleWithRequestMetadata, + EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}")); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index c9ec5adc98c..7ca964e06dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index e5757604caf..0df94598935 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index b7adc3064fa..da770c9c023 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -65,6 +65,8 @@ public class ControllerContainerTest { " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + " </rotations>\n" + " </config>\n" + + " " + + "<accesslog type='disabled'/>\n" + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e96d25c2cab..0f846ab57e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1610,7 +1610,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { var changedAt = tester.controller().clock().instant(); - var westPolicies = tester.controller().applications().routingPolicies().get(deployment); + var westPolicies = tester.controller().routingController().policies().get(deployment); assertEquals(1, westPolicies.size()); var westPolicy = westPolicies.values().iterator().next(); assertEquals(status, westPolicy.status().globalRouting().status()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index 9ef94cc9afd..57774c3f412 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 22ba83a4730..10201af0272 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -8,7 +8,20 @@ { "cluster": "default", "tls": true, - "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/" + "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", + "scope": "zone" + }, + { + "cluster": "default", + "tls": true, + "url": "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/", + "scope": "global" + }, + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa:43", + "scope": "zone" } ], "serviceUrls": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index f9692a4afb7..d96b291234d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -4,7 +4,20 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "scope": "zone" + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 204927f6954..3ff8533fdb3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -4,7 +4,14 @@ "instance": "instance1", "environment": "dev", "region": "us-east-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa:43", + "scope": "zone" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-east-1.dev.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 39824e22928..bff8326fdeb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -7,7 +7,20 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "scope": "zone" + }, + { + "cluster": "foo", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global" + } + ], "serviceUrls": [ "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index dd3d16fc721..9e7eeba8420 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -83,36 +83,6 @@ { "at": "(ignore)", "type": "info", - "message": "Endpoints not yet ready." - }, - { - "at": "(ignore)", - "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" - }, - { - "at": "(ignore)", - "type": "info", - "message": "--- platform 6.1" - }, - { - "at": "(ignore)", - "type": "info", - "message": "Found endpoints:" - }, - { - "at": "(ignore)", - "type": "info", - "message": "- test.us-east-1" - }, - { - "at": "(ignore)", - "type": "info", - "message": " |-- https://instance1-t--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" - }, - { - "at": "(ignore)", - "type": "info", "message": "Tester container successfully installed!" } ], @@ -224,7 +194,7 @@ } ] }, - "lastId": 40, + "lastId": 34, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index e5670139b0f..635adc73d1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.routing; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -34,14 +35,13 @@ public class RoutingApiTest extends ControllerContainerTest { } @Test - public void policy_based_routing() { + public void exclusive_routing() { var context = deploymentTester.newDeploymentContext(); // Zones support direct routing var westZone = ZoneId.from("prod", "us-west-1"); var eastZone = ZoneId.from("prod", "us-east-3"); - deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone), - ZoneApiMock.from(eastZone)); - + deploymentTester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(westZone), + ZoneApiMock.from(eastZone)); // Deploy application var applicationPackage = new ApplicationPackageBuilder() .region(westZone.region()) @@ -96,7 +96,7 @@ public class RoutingApiTest extends ControllerContainerTest { } @Test - public void rotation_based_routing() { + public void shared_routing() { // Deploy application var context = deploymentTester.newDeploymentContext(); var westZone = ZoneId.from("prod", "us-west-1"); @@ -156,10 +156,16 @@ public class RoutingApiTest extends ControllerContainerTest { // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation @Test public void mixed_routing() { - // Deploy application - var context = deploymentTester.newDeploymentContext(); var westZone = ZoneId.from("prod", "us-west-1"); var eastZone = ZoneId.from("prod", "us-east-3"); + + // One zone supports multiple routing methods + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), + RoutingMethod.shared, + RoutingMethod.exclusive); + + // Deploy application + var context = deploymentTester.newDeploymentContext(); var applicationPackage = new ApplicationPackageBuilder() .region(westZone.region()) .region(eastZone.region()) @@ -168,7 +174,6 @@ public class RoutingApiTest extends ControllerContainerTest { context.submit(applicationPackage).deploy(); // Assign policy in one zone - deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone)); context.addRoutingPolicy(westZone, true); // GET status with both policy and rotation assigned diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json index b3f3e90a9cd..1c23c6bb569 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json index cdf3e093f97..eea78c1b963 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json index 373d7076ffc..6cb90bdb673 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json index 74c8173d132..59519c33d06 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json index 444b6a825ea..e95d9bcdc42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json index 3f5353505df..49b85775e63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json index 376b6c9c902..abf0a46ae3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json index 482fd920070..8328e1ffab1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json index 5ab261067bf..d86ca2d56e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "out", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json index fd30a27fba5..5b15b72752c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json index 59ea77cf7ae..90b2317c1b3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json index 0240079f154..85e345c01d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json index a883fd3f342..eb06e9ee11d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json index a883fd3f342..eb06e9ee11d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json index 18803f56de3..440b80bc4d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "out", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 674f084a8b7..1bb531edbc5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -55,7 +55,7 @@ public class RotationRepositoryTest { @Before public void before() { tester = new DeploymentTester(new ControllerTester(rotationsConfig)); - repository = tester.applications().rotationRepository(); + repository = tester.controller().routingController().rotations(); application = tester.newDeploymentContext("tenant1", "app1", "default"); } @@ -83,7 +83,7 @@ public class RotationRepositoryTest { @Test public void strips_whitespace_in_rotation_fqdn() { tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); - RotationRepository repository = tester.controller().applications().rotationRepository(); + RotationRepository repository = tester.controller().routingController().rotations(); var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); application2.submit(applicationPackage); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index c6f9bb81b63..89b7e13e392 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -8,19 +8,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -253,7 +259,7 @@ public class RoutingPoliciesTest { URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), ClusterSpec.Id.from("c2"), URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), - tester.controllerTester().controller().applications().clusterEndpoints(context.deploymentIdIn(zone1))); + tester.controllerTester().controller().routingController().zoneEndpointsOf(context.deploymentIdIn(zone1))); } @Test @@ -266,7 +272,7 @@ public class RoutingPoliciesTest { var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) - .setDirectlyRouted(zoneApi); + .exclusiveRoutingIn(zoneApi); tester.provisionLoadBalancers(1, context.instanceId(), zone); // Deploy to dev @@ -289,7 +295,7 @@ public class RoutingPoliciesTest { var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) - .setDirectlyRouted(zoneApi); + .exclusiveRoutingIn(zoneApi); var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); assertEquals(prodRecords, tester.recordNames()); @@ -484,6 +490,48 @@ public class RoutingPoliciesTest { tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } + + @Test + public void non_production_deployment_is_not_registered_in_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + + // Configure the system to use the same region for test, staging and prod + var sharedRegion = RegionName.from("aws-us-east-1c"); + var prodZone = ZoneId.from(Environment.prod, sharedRegion); + var stagingZone = ZoneId.from(Environment.staging, sharedRegion); + var testZone = ZoneId.from(Environment.test, sharedRegion); + var zones = List.of(ZoneApiMock.from(prodZone), + ZoneApiMock.from(stagingZone), + ZoneApiMock.from(testZone)); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), + SystemApplication.all()); + + var context = tester.tester.newDeploymentContext(); + var endpointId = EndpointId.of("r0"); + var applicationPackage = new ApplicationPackageBuilder() + .region(sharedRegion) + .endpoint(endpointId.id(), "default") + .build(); + + // Application starts deployment + context = context.submit(applicationPackage); + for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + context = context.runJob(testJob); + // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the + // deployment, we consume only one DNS update at a time here + do { + context = context.flushDnsUpdates(1); + tester.assertTargets(context.instanceId(), endpointId, 0); + } while (!tester.recordNames().isEmpty()); + } + + // Deployment completes + context.completeRollout(); + tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); + } private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); @@ -505,11 +553,15 @@ public class RoutingPoliciesTest { private final DeploymentTester tester; public RoutingPoliciesTester() { - this(new DeploymentTester()); + this(SystemName.main); + } + + public RoutingPoliciesTester(SystemName system) { + this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system)))); } public RoutingPolicies routingPolicies() { - return tester.controllerTester().controller().applications().routingPolicies(); + return tester.controllerTester().controller().routingController().policies(); } public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { @@ -523,7 +575,7 @@ public class RoutingPoliciesTest { public RoutingPoliciesTester(DeploymentTester tester) { this.tester = tester; // Make all zones directly routed - tester.controllerTester().zoneRegistry().setDirectlyRouted(Set.copyOf(tester.controllerTester().zoneRegistry().zones().all().zones())); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { @@ -559,12 +611,7 @@ public class RoutingPoliciesTest { } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var prefix = ""; - if (!endpointId.equals(EndpointId.defaultId())) { - prefix = endpointId.id() + "."; - } - var endpoint = prefix + application.application().value() + "." + application.tenant().value() + - ".global.vespa.oath.cloud"; + var endpoint = RoutingPolicy.globalEndpointOf(application, endpointId, tester.controller().system()).dnsName(); var zoneTargets = Arrays.stream(zone) .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value()) diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index 769c4323b0c..125c5004d25 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -15,51 +15,6 @@ <binding>http://*/tester/v1/*</binding> </handler> - <http> - <!-- Make sure 4080 is the first port. This will be used by the config server. --> - <server id='default' port='4080'/> - <server id='testertls4443' port='4443'> - <config name="jdisc.http.connector"> - <tlsClientAuthEnforcer> - <enable>true</enable> - <pathWhitelist> - <item>/status.html</item> - <item>/state/v1/config</item> - </pathWhitelist> - </tlsClientAuthEnforcer> - </config> - <ssl> - <private-key-file>/var/lib/sia/keys/vespa.vespa.tenant.key.pem</private-key-file> - <certificate-file>/var/lib/sia/certs/vespa.vespa.tenant.cert.pem</certificate-file> - <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file> - <client-authentication>want</client-authentication> - </ssl> - </server> - <filtering> - <access-control domain='vespa.vespa.cd'> - <exclude> - <binding>http://*/tester/v1/*</binding> - </exclude> - </access-control> - <request-chain id="testrunner-api"> - <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle="jdisc-security-filters"> - <config name="jdisc.http.filter.security.athenz.athenz-authorization-filter"> - <credentialsToVerify>TOKEN_ONLY</credentialsToVerify> - <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName> - </config> - <component id="com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper" bundle="jdisc-security-filters"> - <config name="jdisc.http.filter.security.athenz.static-request-resource-mapper"> - <resourceName>vespa.vespa.cd:tester-application</resourceName> - <action>deploy</action> - </config> - </component> - </filter> - </request-chain> - </filtering> - </http> - - <accesslog type='json' fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S'/> - <nodes count="1" allocated-memory="17%"> <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> </nodes> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 9b7c7302a54..d6781f14e75 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -4,12 +4,13 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_rhel_6_10) message("-- Setting up default build settings for rhel 6.10") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_rhel_7_7) message("-- Setting up default build settings for rhel 7.7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) endfunction() @@ -21,12 +22,13 @@ endfunction() function(setup_vespa_default_build_settings_centos_7) message("-- Setting up default build settings for centos 7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() @@ -39,14 +41,14 @@ function(setup_vespa_default_build_settings_darwin) else() set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endif() - set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" PARENT_SCOPE) - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib") + set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" PARENT_SCOPE) + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY) list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}") endif() list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include") if(DEFINED DEFAULT_LLVM_INCLUDE_DIRECTORY) list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_LLVM_INCLUDE_DIRECTORY}") endif() @@ -56,21 +58,25 @@ endfunction() function(setup_vespa_default_build_settings_fedora_29) message("-- Setting up default build settings for fedora 29") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_30) message("-- Setting up default build settings for fedora 30") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_31) message("-- Setting up default build settings for fedora 31") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_32) message("-- Setting up default build settings for fedora 32") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() @@ -81,9 +87,45 @@ function(setup_vespa_default_build_settings_ubuntu_18_10) set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE) endfunction() +function(vespa_use_default_vespa_unprivileged) + if(NOT DEFINED VESPA_UNPRIVILEGED) + message("-- Setting VESPA_UNPRIVILEGED to yes") + set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE) + endif() +endfunction() + +function(vespa_use_default_cmake_install_prefix) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa") + if(COMMAND vespa_use_specific_install_prefix) + vespa_use_specific_install_prefix() + endif() + else() + set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa") + endif() + message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") + set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE) + endif() +endfunction() + +function(vespa_use_default_vespa_user) + if(NOT DEFINED VESPA_USER) + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_VESPA_USER "vespa") + if(COMMAND vespa_use_specific_vespa_user) + vespa_use_specific_vespa_user() + endif() + else() + set(DEFAULT_VESPA_USER "$ENV{USER}") + endif() + message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}") + set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE) + endif() +endfunction() + function(vespa_use_default_build_settings) - if (DEFINED CMAKE_INSTALL_PREFIX AND DEFINED CMAKE_PREFIX_PATH AND - NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND + if (DEFINED CMAKE_PREFIX_PATH AND DEFINED VESPA_LLVM_VERSION AND DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH) @@ -94,26 +136,7 @@ function(vespa_use_default_build_settings) unset(DEFAULT_CMAKE_PREFIX_PATH) unset(DEFAULT_EXTRA_LINK_DIRECTORY) unset(DEFAULT_EXTRA_INCLUDE_DIRECTORY) - unset(DEFAULT_VESPA_USER) unset(DEFAULT_VESPA_CPU_ARCH_FLAGS) - if(NOT DEFINED VESPA_UNPRIVILEGED) - message("-- Setting VESPA_UNPRIVILEGED to yes") - set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE) - set(VESPA_UNPRIVILEGED "yes") - endif() - if(VESPA_UNPRIVILEGED STREQUAL "no") - set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa") - set(DEFAULT_VESPA_USER "vespa") - if(COMMAND vespa_use_specific_install_prefix) - vespa_use_specific_install_prefix() - endif() - if(COMMAND vespa_use_specific_vespa_user) - vespa_use_specific_vespa_user() - endif() - else() - set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa") - set(DEFAULT_VESPA_USER "$ENV{USER}") - endif() if(APPLE) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(VESPA_DEPS "/opt/vespa-deps-clang") @@ -174,9 +197,6 @@ function(vespa_use_default_build_settings) set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-mtune=intel") endif() endif() - if(DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) - message("-- DEFAULT_CMAKE_INSTALL_PREFIX is ${DEFAULT_CMAKE_INSTALL_PREFIX}") - endif() if(DEFINED DEFAULT_CMAKE_PREFIX_PATH) message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") endif() @@ -189,14 +209,7 @@ function(vespa_use_default_build_settings) if(DEFINED DEFAULT_VESPA_LLVM_VERSION) message("-- DEFAULT_VESPA_LLVM_VERSION is ${DEFAULT_VESPA_LLVM_VERSION}") endif() - if(DEFINED DEFAULT_VESPA_USER) - message("-- DEFAULT_VESPA_USER is ${DEFAULT_VESPA_USER}") - endif() message("-- DEFAULT_VESPA_CPU_ARCH_FLAGS is ${DEFAULT_VESPA_CPU_ARCH_FLAGS}") - if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) - message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") - set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE) - endif() if(NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED DEFAULT_CMAKE_PREFIX_PATH) message("-- Setting CMAKE_PREFIX_PATH to ${DEFAULT_CMAKE_PREFIX_PATH}") set(CMAKE_PREFIX_PATH "${DEFAULT_CMAKE_PREFIX_PATH}" PARENT_SCOPE) @@ -237,10 +250,6 @@ function(vespa_use_default_build_settings) message("-- Setting VESPA_LLVM_VERSION to ${DEFAULT_VESPA_LLVM_VERSION}") set(VESPA_LLVM_VERSION "${DEFAULT_VESPA_LLVM_VERSION}" PARENT_SCOPE) endif() - if(NOT DEFINED VESPA_USER AND DEFINED DEFAULT_VESPA_USER) - message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}") - set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE) - endif() if(NOT DEFINED VESPA_CPU_ARCH_FLAGS AND DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS) message("-- Setting VESPA_CPU_ARCH_FLAGS to ${DEFAULT_VESPA_CPU_ARCH_FLAGS}") set(VESPA_CPU_ARCH_FLAGS "${DEFAULT_VESPA_CPU_ARCH_FLAGS}" PARENT_SCOPE) diff --git a/dist/vespa.spec b/dist/vespa.spec index 833e4798084..3a336496f4c 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -47,10 +47,11 @@ BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: vespa-openssl-devel >= 1.1.1c-1 +BuildRequires: vespa-icu-devel >= 65.1.0-1 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 -BuildRequires: llvm-devel >= 7.0.1-3 +BuildRequires: llvm-devel >= 8.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: openssl-devel BuildRequires: vespa-gtest >= 1.8.1-1 @@ -91,7 +92,9 @@ BuildRequires: openblas-devel BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel +%if ! 0%{?el7} BuildRequires: libicu-devel +%endif BuildRequires: java-11-openjdk-devel BuildRequires: rpm-build BuildRequires: make @@ -132,25 +135,28 @@ Requires: openblas-serial Requires: lz4 Requires: libzstd Requires: zlib +%if ! 0%{?el7} Requires: libicu +%endif Requires: perf Requires: gdb Requires: net-tools %if 0%{?el7} Requires: llvm5.0 Requires: vespa-openssl >= 1.1.1c-1 +Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 %define _vespa_llvm_version 5.0 %define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64 -%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include +%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?el8} -Requires: llvm-libs >= 7.0.1-3 +Requires: llvm-libs >= 8.0.1 Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs -%define _vespa_llvm_version 7.0 +%define _vespa_llvm_version 8 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 -%define _extra_include_directory %{_vespa_deps_prefix}/include +%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} Requires: vespa-protobuf >= 3.7.0-4 @@ -172,7 +178,7 @@ Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 -%define _extra_include_directory %{_vespa_deps_prefix}/include +%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif Requires: java-11-openjdk Requires(pre): shadow-utils diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java index ffd5dffece9..c7b9d342043 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java @@ -124,4 +124,8 @@ public class ContainerResources { public ContainerResources withMemoryBytes(long memoryBytes) { return new ContainerResources(cpus, cpuShares, memoryBytes); } + + public ContainerResources withUnlimitedCpus() { + return new ContainerResources(0, 0, memoryBytes); + } } diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java index 6868bc8ecd4..4160f366fdb 100644 --- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -37,8 +37,8 @@ public class ProcessingUpdateTestCase { @Test public void testProcessingUpdates() { DocumentType articleType = new DocumentType("article"); - Field bodyField = new Field("body", DataType.STRING, true); - Field titleField = new Field("title", DataType.STRING, true); + Field bodyField = new Field("body", DataType.STRING); + Field titleField = new Field("title", DataType.STRING); articleType.addField(bodyField); articleType.addField(titleField); dtm = new DocumentTypeManager(); diff --git a/document/abi-spec.json b/document/abi-spec.json index e4917a174a6..abbf03bc228 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -427,6 +427,8 @@ "methods": [ "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)", + "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.document.DocumentType clone()", "public com.yahoo.document.Document createFieldValue()", "public java.lang.Class getValueClass()", @@ -448,6 +450,8 @@ "public com.yahoo.document.Field getField(int)", "public boolean hasField(java.lang.String)", "public int getFieldCount()", + "public java.util.Set getImportedFieldNames()", + "public boolean hasImportedField(java.lang.String)", "public com.yahoo.document.Field removeField(java.lang.String)", "public java.util.Collection getFields()", "public java.util.Set fieldSet()", @@ -712,6 +716,8 @@ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)", "public com.yahoo.document.DocumenttypesConfig$Documenttype build()" ], "fields": [ @@ -719,7 +725,8 @@ "public java.util.List datatype", "public java.util.List annotationtype", "public java.util.Map fieldsets", - "public java.util.List referencetype" + "public java.util.List referencetype", + "public java.util.List importedfield" ] }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": { @@ -1264,6 +1271,35 @@ ], "fields": [] }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()" + ], + "fields": [] + }, + "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)", + "public java.lang.String name()" + ], + "fields": [] + }, "com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -1347,7 +1383,9 @@ "public java.util.Map fieldsets()", "public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)", "public java.util.List referencetype()", - "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)" + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)", + "public java.util.List importedfield()", + "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)" ], "fields": [] }, @@ -1457,8 +1495,10 @@ ], "methods": [ "public void <init>(java.lang.String, int, com.yahoo.document.DataType, boolean)", + "public void <init>(java.lang.String, int, com.yahoo.document.DataType)", "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean, com.yahoo.document.DocumentType)", + "public void <init>(java.lang.String, com.yahoo.document.DataType, com.yahoo.document.DocumentType)", "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean)", "public void <init>(java.lang.String, com.yahoo.document.DataType)", "public void <init>(java.lang.String, com.yahoo.document.Field)", diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index f3ff6e4ed30..23559878fbb 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType { private StructDataType bodyType; private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); + private final Set<String> importedFieldNames; /** * Creates a new document type and registers it with the document type manager. @@ -51,8 +52,7 @@ public class DocumentType extends StructuredDataType { * @param name The name of the new document type */ public DocumentType(String name) { - this(name, new StructDataType(name + ".header"), - new StructDataType(name + ".body")); + this(name, createHeaderStructType(name), createBodyStructType(name)); } /** @@ -65,9 +65,27 @@ public class DocumentType extends StructuredDataType { * @param bodyType The type of the body struct */ public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { + this(name, headerType, bodyType, Collections.emptySet()); + } + + public DocumentType(String name, StructDataType headerType, + StructDataType bodyType, Set<String> importedFieldNames) { super(name); this.headerType = headerType; this.bodyType = bodyType; + this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); + } + + public DocumentType(String name, Set<String> importedFieldNames) { + this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames); + } + + private static StructDataType createHeaderStructType(String name) { + return new StructDataType(name + ".header"); + } + + private static StructDataType createBodyStructType(String name) { + return new StructDataType(name + ".body"); } @Override @@ -181,8 +199,7 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } - StructDataType struct = (field.isHeader() ? headerType : bodyType); - struct.addField(field); + headerType.addField(field); } // Do not use, public only for testing @@ -221,8 +238,8 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } - Field field = new Field(name, type, false); - bodyType.addField(field); + Field field = new Field(name, type); + headerType.addField(field); return field; } @@ -234,13 +251,9 @@ public class DocumentType extends StructuredDataType { * @return The field created * TODO Fix searchdefinition so that exception can be thrown if filed is already registerd */ + @Deprecated public Field addHeaderField(String name, DataType type) { - if (isRegistered()) { - throw new IllegalStateException("You cannot add fields to a document type that is already registered."); - } - Field field = new Field(name, type, true); - headerType.addField(field); - return field; + return addField(name, type); } /** @@ -374,6 +387,14 @@ public class DocumentType extends StructuredDataType { return headerType.getFieldCount() + bodyType.getFieldCount(); } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + + public boolean hasImportedField(String fieldName) { + return importedFieldNames.contains(fieldName); + } + /** * Removes an field from the DocumentType. * diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 154c25880a9..21a163aa2b9 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -10,8 +10,10 @@ import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Configures the Vespa document manager from a config id. @@ -144,7 +146,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (Field field : body.getFields()) { field.setHeader(false); } - DocumentType type = new DocumentType(doc.name(), header, body); + var importedFields = doc.importedfield().stream() + .map(f -> f.name()) + .collect(Collectors.toUnmodifiableSet()); + DocumentType type = new DocumentType(doc.name(), header, body, importedFields); for (Object j : doc.inherits()) { DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; @@ -180,9 +185,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub : manager.getDataType(field.datatype(), field.detailedtype()); if (field.id().size() == 1) { - type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); + type.addField(new Field(field.name(), field.id().get(0).id(), fieldType)); } else { - type.addField(new Field(field.name(), fieldType, true)); + type.addField(new Field(field.name(), fieldType)); } } manager.register(type); diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 671c8c7f763..9be4036174c 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -21,7 +21,6 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab protected DataType dataType; protected int fieldId; - private boolean isHeader; private boolean forcedId; /** @@ -32,11 +31,14 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param isHeader Whether this is a "header" field or a "content" field * (true = "header"). */ + @Deprecated public Field(String name, int id, DataType dataType, boolean isHeader) { + this(name, id, dataType); + } + public Field(String name, int id, DataType dataType) { super(name); this.fieldId = id; this.dataType = dataType; - this.isHeader = isHeader; this.forcedId = true; validateId(id, null); } @@ -55,8 +57,13 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * (true = "header"). * @param owner the owning document (used to check for id collisions) */ + @Deprecated public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) { - this(name, 0, dataType, isHeader); + this(name, dataType, owner); + } + + public Field(String name, DataType dataType, DocumentType owner) { + this(name, 0, dataType); this.fieldId = calculateIdV7(owner); this.forcedId = false; } @@ -69,8 +76,9 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param isHeader Whether this is a "header" field or a "content" field * (true = "header"). */ + @Deprecated public Field(String name, DataType dataType, boolean isHeader) { - this(name, dataType, isHeader, null); + this(name, dataType); } /** @@ -80,7 +88,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param dataType The datatype of the field */ public Field(String name, DataType dataType) { - this(name, dataType, true); + this(name, dataType, null); } /** @@ -89,7 +97,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab */ // TODO: Decide on one copy/clone idiom and do it for this and all it is calling public Field(String name, Field field) { - this(name, field.dataType, field.isHeader, null); + this(name, field.dataType, null); } public int compareTo(Object o) { @@ -196,14 +204,12 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ @Deprecated // TODO: Remove on Vespa 8 public boolean isHeader() { - return isHeader; + return true; } /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ @Deprecated // TODO: Remove on Vespa 8 - public void setHeader(boolean header) { - this.isHeader = header; - } + public void setHeader(boolean header) { } /** Two fields are equal if they have the same name and the same data type */ @Override diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java index 295b1c9dd5c..134681a4028 100755 --- a/document/src/main/java/com/yahoo/document/FieldPath.java +++ b/document/src/main/java/com/yahoo/document/FieldPath.java @@ -10,7 +10,7 @@ import java.util.List; * This class represents a path into a document, that can be used to iterate through the document and extract the field * values you're interested in. * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public class FieldPath implements Iterable<FieldPathEntry> { diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index 0cedda7c4f0..9e2759e1590 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -7,6 +7,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentGet; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.FieldPath; import com.yahoo.document.datatypes.FieldPathIteratorHandler; @@ -131,10 +132,37 @@ public class AttributeNode implements ExpressionNode { throw new IllegalStateException("Function '" + function + "' is not supported."); } - private static Object evaluateFieldPath(String fieldPth, Object value) { + private static boolean looksLikeComplexFieldPath(String path) { + for (int i = 0; i < path.length(); ++i) { + switch (path.charAt(i)) { + case '.': + case '{': + case '[': + return true; + } + } + return false; + } + + private static boolean isSimpleImportedField(String path, DocumentType documentType) { + if (looksLikeComplexFieldPath(path)) { + return false; + } + return documentType.hasImportedField(path); + } + + private static Object evaluateFieldPath(String fieldPathStr, Object value) { if (value instanceof DocumentPut) { final Document doc = ((DocumentPut) value).getDocument(); - FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth); + if (isSimpleImportedField(fieldPathStr, doc.getDataType())) { + // Imported fields can only be meaningfully evaluated in the backend, so we + // explicitly treat them as if they are valid fields with missing values. This + // will be treated the same as if it's a normal field by the selection operators. + // This avoids any awkward interaction with Invalid values or having to + // augment the FieldPath code with knowledge of imported fields. + return null; + } + FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPathStr); IteratorHandler handler = new IteratorHandler(); doc.iterateNested(fieldPath, 0, handler); if (handler.values.isEmpty()) { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index c2111edfd10..630f204c44d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -82,7 +82,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume } public void write(Document doc) { - write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); + write(new Field(doc.getDataType().getName(), 0, doc.getDataType()), doc); } @SuppressWarnings("deprecation") diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 0d6b0cae926..5db98f26141 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -320,7 +320,7 @@ public final class XmlDocumentWriter implements DocumentWriter { buffer = new XmlStream(); buffer.setIndent(indent); optionalWrapperMarker.clear(); - write(new Field(document.getDataType().getName(), 0, document.getDataType(), true), document); + write(new Field(document.getDataType().getName(), 0, document.getDataType()), document); } @Override diff --git a/document/src/test/document/documentmanager.importedfields.cfg b/document/src/test/document/documentmanager.importedfields.cfg new file mode 100644 index 00000000000..765e290ae82 --- /dev/null +++ b/document/src/test/document/documentmanager.importedfields.cfg @@ -0,0 +1,65 @@ +datatype[7] +datatype[0].id -1365874599 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name referenced_type.header +datatype[0].structtype[0].version 9 +datatype[0].structtype[0].field[0] +datatype[0].documenttype[0] +datatype[1].id 278604398 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name referenced_type.body +datatype[1].structtype[0].version 9 +datatype[1].structtype[0].field[0] +datatype[1].documenttype[0] +datatype[2].id 124647170 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name referenced_type +datatype[2].documenttype[0].version 9 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct -1365874599 +datatype[2].documenttype[0].bodystruct 278604398 +datatype[3].id 12345678 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[3].referencetype[1] +datatype[3].referencetype[0].target_type_id 124647170 +datatype[4].id 673066331 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name type_with_ref.header +datatype[4].structtype[0].version 234 +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name my_ref_field +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 12345678 +datatype[4].documenttype[0] +datatype[5].id -176986064 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[1] +datatype[5].structtype[0].name type_with_ref.body +datatype[5].structtype[0].version 234 +datatype[5].structtype[0].field[0] +datatype[5].documenttype[0] +datatype[6].id -1293964543 +datatype[6].arraytype[0] +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[1] +datatype[6].documenttype[0].name type_with_ref +datatype[6].documenttype[0].version 234 +datatype[6].documenttype[0].inherits[0] +datatype[6].documenttype[0].headerstruct 673066331 +datatype[6].documenttype[0].bodystruct -176986064 +datatype[6].documenttype[0].importedfield[0].name "my_cool_imported_field" +datatype[6].documenttype[0].importedfield[1].name "my_awesome_imported_field"
\ No newline at end of file diff --git a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java index fb2d478d38b..36cc18ebd6b 100755 --- a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java @@ -27,11 +27,11 @@ public class DocumentCalculatorTestCase { docMan = new DocumentTypeManager(); testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("byteattr", DataType.BYTE); - testDocType.addHeaderField("intattr", DataType.INT); - testDocType.addHeaderField("longattr", DataType.LONG); - testDocType.addHeaderField("doubleattr", DataType.DOUBLE); - testDocType.addHeaderField("missingattr", DataType.INT); + testDocType.addField("byteattr", DataType.BYTE); + testDocType.addField("intattr", DataType.INT); + testDocType.addField("longattr", DataType.LONG); + testDocType.addField("doubleattr", DataType.DOUBLE); + testDocType.addField("missingattr", DataType.INT); docMan.registerDocumentType(testDocType); doc = new Document(testDocType, new DocumentId("id:ns:testdoc::testdoc:http://www.ntnu.no/")); diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java index 08abd6e6a2d..fea3b265b6d 100644 --- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java @@ -34,11 +34,11 @@ public class DocumentIdTestCase { public void setUp() { DocumentType testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("intattr", DataType.INT); + testDocType.addField("intattr", DataType.INT); testDocType.addField("rawattr", DataType.RAW); testDocType.addField("floatattr", DataType.FLOAT); - testDocType.addHeaderField("stringattr", DataType.STRING); - testDocType.addHeaderField("Minattr", DataType.INT); + testDocType.addField("stringattr", DataType.STRING); + testDocType.addField("Minattr", DataType.INT); manager.registerDocumentType(testDocType); } diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index bc1224ca8ea..fa47c80c6fb 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -62,25 +62,25 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { public void testSerializationAllVersions() throws IOException { DocumentType docInDocType = new DocumentType("docindoc"); - docInDocType.addField(new Field("stringindocfield", DataType.STRING, false)); + docInDocType.addField(new Field("stringindocfield", DataType.STRING)); DocumentType docType = new DocumentType("serializetest"); - docType.addField(new Field("floatfield", DataType.FLOAT, true)); - docType.addField(new Field("stringfield", DataType.STRING, true)); - docType.addField(new Field("longfield", DataType.LONG, true)); - docType.addField(new Field("urifield", DataType.URI, true)); - docType.addField(new Field("intfield", DataType.INT, false)); - docType.addField(new Field("rawfield", DataType.RAW, false)); - docType.addField(new Field("doublefield", DataType.DOUBLE, false)); - docType.addField(new Field("bytefield", DataType.BYTE, false)); - docType.addField(new Field("boolfield", DataType.BOOL, false)); + docType.addField(new Field("floatfield", DataType.FLOAT)); + docType.addField(new Field("stringfield", DataType.STRING)); + docType.addField(new Field("longfield", DataType.LONG)); + docType.addField(new Field("urifield", DataType.URI)); + docType.addField(new Field("intfield", DataType.INT)); + docType.addField(new Field("rawfield", DataType.RAW)); + docType.addField(new Field("doublefield", DataType.DOUBLE)); + docType.addField(new Field("bytefield", DataType.BYTE)); + docType.addField(new Field("boolfield", DataType.BOOL)); DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT); - docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false)); + docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType)); DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType); - docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false)); - docType.addField(new Field("docfield", DataType.DOCUMENT, false)); + docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType)); + docType.addField(new Field("docfield", DataType.DOCUMENT)); DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false); - docType.addField(new Field("wsfield", weightedSetDataType, false)); + docType.addField(new Field("wsfield", weightedSetDataType)); DocumentTypeManager docMan = new DocumentTypeManager(); docMan.register(docInDocType); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 141a74a24fe..dcd4622b3f4 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -107,26 +107,26 @@ public class DocumentTestCase extends DocumentTestCaseBase { docMan = new DocumentTypeManager(); DocumentType docInDocType = new DocumentType("docindoc"); - docInDocType.addField(new Field("tull", 2, docMan.getDataType(2), true)); + docInDocType.addField(new Field("tull", 2, docMan.getDataType(2))); docMan.registerDocumentType(docInDocType); DocumentType sertestDocType = new DocumentType("sertest"); - sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("date", 3, docMan.getDataType(0), true)); - sertestDocType.addField(new Field("from", 4, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("to", 6, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2), true)); - sertestDocType.addField(new Field("body", 10, docMan.getDataType(2), false)); - sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0), false)); - sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)), false)); - sertestDocType.addField(new Field("rawfield", 879, DataType.RAW, false)); - sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING), false)); - sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false), false)); - sertestDocType.addField(new Field("docindoc", 882, docInDocType, false)); - sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false)); - sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false)); - sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL, false)); + sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2))); + sertestDocType.addField(new Field("date", 3, docMan.getDataType(0))); + sertestDocType.addField(new Field("from", 4, docMan.getDataType(2))); + sertestDocType.addField(new Field("to", 6, docMan.getDataType(2))); + sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2))); + sertestDocType.addField(new Field("body", 10, docMan.getDataType(2))); + sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0))); + sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)))); + sertestDocType.addField(new Field("rawfield", 879, DataType.RAW)); + sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING))); + sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false))); + sertestDocType.addField(new Field("docindoc", 882, docInDocType)); + sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING))); + sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE)); + sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL)); docMan.registerDocumentType(sertestDocType); } @@ -880,13 +880,13 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testInheritance() { // Create types that inherit each other.. And test that it works.. DocumentType parentType = new DocumentType("parent"); - parentType.addField(new Field("parentbodyint", DataType.INT, false)); - parentType.addField(new Field("parentheaderint", DataType.INT, true)); - parentType.addField(new Field("overwritten", DataType.INT, true)); + parentType.addField(new Field("parentbodyint", DataType.INT)); + parentType.addField(new Field("parentheaderint", DataType.INT)); + parentType.addField(new Field("overwritten", DataType.INT)); DocumentType childType = new DocumentType("child"); - childType.addField(new Field("childbodyint", DataType.INT, false)); - childType.addField(new Field("childheaderint", DataType.INT, true)); - childType.addField(new Field("overwritten", DataType.INT, true)); + childType.addField(new Field("childbodyint", DataType.INT)); + childType.addField(new Field("childheaderint", DataType.INT)); + childType.addField(new Field("overwritten", DataType.INT)); childType.inherit(parentType); DocumentTypeManager manager = new DocumentTypeManager(); @@ -914,13 +914,13 @@ public class DocumentTestCase extends DocumentTestCaseBase { @Test public void testInheritanceTypeMismatch() { DocumentType parentType = new DocumentType("parent"); - parentType.addField(new Field("parentbodyint", DataType.INT, false)); - parentType.addField(new Field("parentheaderint", DataType.INT, true)); - parentType.addField(new Field("overwritten", DataType.STRING, true)); + parentType.addField(new Field("parentbodyint", DataType.INT)); + parentType.addField(new Field("parentheaderint", DataType.INT)); + parentType.addField(new Field("overwritten", DataType.STRING)); DocumentType childType = new DocumentType("child"); - childType.addField(new Field("childbodyint", DataType.INT, false)); - childType.addField(new Field("childheaderint", DataType.INT, true)); - childType.addField(new Field("overwritten", DataType.INT, true)); + childType.addField(new Field("childbodyint", DataType.INT)); + childType.addField(new Field("childheaderint", DataType.INT)); + childType.addField(new Field("overwritten", DataType.INT)); try { childType.inherit(parentType); fail("Inheritance with conflicting types worked."); @@ -934,7 +934,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testFieldValueImplementations() { docMan = new DocumentTypeManager(); DocumentType docType = new DocumentType("impl"); - docType.addField(new Field("something", DataType.getArray(DataType.STRING), false)); + docType.addField(new Field("something", DataType.getArray(DataType.STRING))); docMan.register(docType); //just checks that isAssignableFrom() in Document.setFieldValue() goes the right way @@ -1276,9 +1276,9 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void testDocumentComparisonDoesNotCorruptStateBug6394548() { DocumentTypeManager docMan = new DocumentTypeManager(); DocumentType docType = new DocumentType("bug2354045"); - docType.addField(new Field("string", 2, DataType.STRING, true)); - docType.addField(new Field("int", 1, DataType.INT, true)); - docType.addField(new Field("float", 0, DataType.FLOAT, true)); + docType.addField(new Field("string", 2, DataType.STRING)); + docType.addField(new Field("int", 1, DataType.INT)); + docType.addField(new Field("float", 0, DataType.FLOAT)); docMan.register(docType); Document doc1 = new Document(docType, new DocumentId("id:ns:bug2354045::bug6394548")); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java index 68fe1c8cc57..6f95f77f08c 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java @@ -23,14 +23,14 @@ public class DocumentTestCaseBase { docMan = new DocumentTypeManager(); testDocType = new DocumentType("testdoc"); - testDocType.addHeaderField("byteattr", DataType.BYTE); - testDocType.addHeaderField("intattr", DataType.INT); + testDocType.addField("byteattr", DataType.BYTE); + testDocType.addField("intattr", DataType.INT); testDocType.addField("rawattr", DataType.RAW); testDocType.addField("floatattr", DataType.FLOAT); - testDocType.addHeaderField("stringattr", DataType.STRING); - testDocType.addHeaderField("Minattr", DataType.INT); - testDocType.addHeaderField("Minattr2", DataType.INT); - testDocType.addHeaderField("primitive1", DataType.INT); + testDocType.addField("stringattr", DataType.STRING); + testDocType.addField("Minattr", DataType.INT); + testDocType.addField("Minattr2", DataType.INT); + testDocType.addField("primitive1", DataType.INT); StructDataType sdt = new StructDataType("struct1"); sdt.addField(new Field("primitive1", DataType.INT)); diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index 65c217e09e1..57b36d9758f 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.StructuredFieldValue; import org.junit.Test; +import java.util.HashSet; import java.util.Iterator; import static org.junit.Assert.assertEquals; @@ -30,7 +31,7 @@ public class DocumentTypeManagerTestCase { DocumentType newDocType = new DocumentType("testdoc"); newDocType.addField("Fjomp", DataType.INT); - newDocType.addHeaderField("Fjols", DataType.STRING); + newDocType.addField("Fjols", DataType.STRING); manager.registerDocumentType(newDocType); @@ -40,7 +41,6 @@ public class DocumentTypeManagerTestCase { assertEquals("Fjomp", fetched4.getName()); assertEquals(fetched4.getDataType(), DataType.INT); - assertEquals(fetched4.isHeader(), false); } @Test @@ -64,8 +64,8 @@ public class DocumentTypeManagerTestCase { StructDataType struct = new StructDataType("mystruct"); DataType wset1 = DataType.getWeightedSet(DataType.getArray(DataType.INT)); DataType wset2 = DataType.getWeightedSet(DataType.getArray(DataType.TAG)); - struct.addField(new Field("foo", wset1, true)); - struct.addField(new Field("bar", wset2, false)); + struct.addField(new Field("foo", wset1)); + struct.addField(new Field("bar", wset2)); DataType array = DataType.getArray(struct); DocumentType docType = new DocumentType("mydoc"); docType.addField("hmm", array); @@ -148,7 +148,6 @@ public class DocumentTypeManagerTestCase { assertTrue(type.hasField("foobarfield1")); Field foobarfield0 = type.getField("foobarfield0"); - assertTrue(!foobarfield0.isHeader()); assertTrue(foobarfield0.getDataType().getCode() == 2); Field foobarfield1 = type.getField("foobarfield1"); @@ -190,7 +189,6 @@ public class DocumentTypeManagerTestCase { assertTrue(type.hasField("arrayarrayfloat")); Field arrayfloat = type.getField("arrayfloat"); - assertTrue(!arrayfloat.isHeader()); ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType(); assertTrue(dataType.getCode() == 99); assertTrue(dataType.getValueClass().equals(Array.class)); @@ -200,7 +198,6 @@ public class DocumentTypeManagerTestCase { Field arrayarrayfloat = type.getField("arrayarrayfloat"); ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType(); - assertTrue(!arrayarrayfloat.isHeader()); assertTrue(subType.getCode() == 4003); assertTrue(subType.getValueClass().equals(Array.class)); assertTrue(subType.getNestedType().getCode() == 99); @@ -218,14 +215,14 @@ public class DocumentTypeManagerTestCase { DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes")); assertNull(banana.getField("newfield")); - assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99), false), customtypes.getField("arrayfloat")); + assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99)), customtypes.getField("arrayfloat")); DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg"); banana = manager.getDocumentType(new DataTypeName("banana")); customtypes = manager.getDocumentType(new DataTypeName("customtypes")); - assertEquals(new Field("newfield", 12345, DataType.STRING, true), banana.getField("newfield")); + assertEquals(new Field("newfield", 12345, DataType.STRING), banana.getField("newfield")); assertNull(customtypes.getField("arrayfloat")); } @@ -559,6 +556,35 @@ search annotationsimplicitstruct { assertTrue(fieldRefType.getTargetType() == targetDocType); } + @Test + public void imported_fields_are_empty_if_no_fields_provided_in_config() { + var manager = createConfiguredManager("file:src/test/document/documentmanager.singlereference.cfg"); + var docType = manager.getDocumentType("type_with_ref"); + + assertNotNull(docType.getImportedFieldNames()); + assertEquals(docType.getImportedFieldNames().size(), 0); + assertFalse(docType.hasImportedField("foo")); + } + + @Test + public void imported_fields_are_populated_from_config() { + var manager = createConfiguredManager("file:src/test/document/documentmanager.importedfields.cfg"); + var docType = manager.getDocumentType("type_with_ref"); + + var expectedFields = new HashSet<String>(); + expectedFields.add("my_cool_imported_field"); + expectedFields.add("my_awesome_imported_field"); + assertEquals(docType.getImportedFieldNames(), expectedFields); + + assertTrue(docType.hasImportedField("my_cool_imported_field")); + assertTrue(docType.hasImportedField("my_awesome_imported_field")); + assertFalse(docType.hasImportedField("a_missing_imported_field")); + } + + // TODO test clone(). Also fieldSets not part of clone()..! + + // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this... + // TODO test reference to own doc type } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index c381a093b4e..9f05a4441b2 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -799,8 +799,8 @@ public class DocumentUpdateTestCase { public TensorUpdateSerializeFixture() { docMan = new DocumentTypeManager(); docType = new DocumentType("test"); - docType.addHeaderField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})"))); - docType.addHeaderField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])"))); + docType.addField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})"))); + docType.addField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])"))); docMan.registerDocumentType(docType); } diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java index 286f7d72b24..7cf0adf6ed1 100644 --- a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java +++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java @@ -17,9 +17,9 @@ public class IncompatibleFieldTypesTest { public void setUp() { arrayOfStrings = new ArrayDataType(DataType.STRING); struct = new StructDataType("fancypants"); - struct.addField(new Field("stringarray", arrayOfStrings, false)); + struct.addField(new Field("stringarray", arrayOfStrings)); DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false); - struct.addField(new Field("stringws", weightedSetOfStrings, false)); + struct.addField(new Field("stringws", weightedSetOfStrings)); root = struct.createFieldValue(); root.setFieldValue("stringarray", arrayOfStrings.createFieldValue()); diff --git a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java index 4197938f0ee..295cfa37e89 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java @@ -20,14 +20,14 @@ public class StructTestCase { @Test public void testBasicStuff() throws Exception { StructDataType type = new StructDataType("teststr"); - type.addField(new Field("int", 0, DataType.INT, true)); - type.addField(new Field("flt", 1, DataType.FLOAT, true)); - type.addField(new Field("str", 2, DataType.STRING, true)); - type.addField(new Field("raw", 3, DataType.RAW, true)); - type.addField(new Field("lng", 4, DataType.LONG, true)); - type.addField(new Field("dbl", 5, DataType.DOUBLE, true)); - type.addField(new Field("uri", 6, DataType.URI, true)); - type.addField(new Field("byt", 8, DataType.BYTE, true)); + type.addField(new Field("int", 0, DataType.INT)); + type.addField(new Field("flt", 1, DataType.FLOAT)); + type.addField(new Field("str", 2, DataType.STRING)); + type.addField(new Field("raw", 3, DataType.RAW)); + type.addField(new Field("lng", 4, DataType.LONG)); + type.addField(new Field("dbl", 5, DataType.DOUBLE)); + type.addField(new Field("uri", 6, DataType.URI)); + type.addField(new Field("byt", 8, DataType.BYTE)); Struct struct = new Struct(type); { @@ -236,7 +236,7 @@ public class StructTestCase { @Test public void testSetUnknownType() { StructDataType type = new StructDataType("teststr"); - type.addField(new Field("int", 0, DataType.INT, true)); + type.addField(new Field("int", 0, DataType.INT)); Struct struct = new Struct(type); try { @@ -251,9 +251,9 @@ public class StructTestCase { public void testCompareToDoesNotMutateStateBug6394548() { StructDataType type = new StructDataType("test"); // NOTE: non-increasing ID order! - type.addField(new Field("int", 2, DataType.INT, true)); - type.addField(new Field("flt", 1, DataType.FLOAT, true)); - type.addField(new Field("str", 0, DataType.STRING, true)); + type.addField(new Field("int", 2, DataType.INT)); + type.addField(new Field("flt", 1, DataType.FLOAT)); + type.addField(new Field("str", 0, DataType.STRING)); Struct a = new Struct(type); a.setFieldValue("int", new IntegerFieldValue(123)); diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java index c11c58ff729..404b069277b 100644 --- a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java +++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java @@ -98,13 +98,13 @@ public class FieldSetTestCase extends DocumentTestCaseBase { assertFalse(new DocIdOnly().contains(headerField)); assertTrue(new HeaderFields().contains(headerField)); - assertFalse(new HeaderFields().contains(bodyField)); + assertTrue(new HeaderFields().contains(bodyField)); assertTrue(new HeaderFields().contains(new DocIdOnly())); assertTrue(new HeaderFields().contains(new NoFields())); - assertContains("[body]", "testdoc:rawattr"); + assertNotContains("[body]", "testdoc:rawattr"); assertContains("[header]", "testdoc:intattr"); - assertNotContains("[header]", "testdoc:rawattr"); + assertContains("[header]", "testdoc:rawattr"); assertContains("testdoc:rawattr,intattr", "testdoc:intattr"); assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr"); assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr"); @@ -141,10 +141,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase { Document doc = getTestDocument(); doc.removeFieldValue("rawattr"); - assertEquals("floatattr:3.56", doCopyFields(doc, "[body]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doCopyFields(doc, "[all]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]")); - assertEquals("byteattr:30,floatattr:3.56", doCopyFields(doc, "testdoc:floatattr,byteattr")); + assertEquals("", doCopyFields(doc, "[body]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[all]")); + assertEquals("floatattr:3.56,byteattr:30", doCopyFields(doc, "testdoc:floatattr,byteattr")); } String doStripFields(Document source, String fieldSet) { @@ -159,10 +159,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase { Document doc = getTestDocument(); doc.removeFieldValue("rawattr"); - assertEquals("floatattr:3.56", doStripFields(doc, "[body]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doStripFields(doc, "[all]")); - assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]")); - assertEquals("byteattr:30,floatattr:3.56", doStripFields(doc, "testdoc:floatattr,byteattr")); + assertEquals("", doStripFields(doc, "[body]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]")); + assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[all]")); + assertEquals("floatattr:3.56,byteattr:30", doStripFields(doc, "testdoc:floatattr,byteattr")); } @Test diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 0ff0bd81d90..0212be8542e 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -8,10 +8,13 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.yolean.Exceptions; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -26,20 +29,24 @@ import static org.junit.Assert.fail; */ public class DocumentSelectorTestCase { + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + private static DocumentTypeManager manager = new DocumentTypeManager(); @Before public void setUp() { - DocumentType type = new DocumentType("test"); - type.addHeaderField("hint", DataType.INT); - type.addHeaderField("hfloat", DataType.FLOAT); - type.addHeaderField("hstring", DataType.STRING); + var importedFields = new HashSet<>(List.of("my_imported_field")); + DocumentType type = new DocumentType("test", importedFields); + type.addField("hint", DataType.INT); + type.addField("hfloat", DataType.FLOAT); + type.addField("hstring", DataType.STRING); type.addField("content", DataType.STRING); StructDataType mystruct = new StructDataType("mystruct"); - mystruct.addField(new Field("key", DataType.INT, false)); - mystruct.addField(new Field("value", DataType.STRING, false)); - type.addHeaderField("mystruct", mystruct); + mystruct.addField(new Field("key", DataType.INT)); + mystruct.addField(new Field("value", DataType.STRING)); + type.addField("mystruct", mystruct); ArrayDataType structarray = new ArrayDataType(mystruct); type.addField("structarray", structarray); @@ -697,6 +704,27 @@ public class DocumentSelectorTestCase { } @Test + public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException { + var documents = createDocs(); + assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.my_imported_field != null", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.my_imported_field", documents.get(0))); + // Only (in)equality operators are well defined for null values; everything else becomes Invalid. + assertEquals(Result.INVALID, evaluate("test.my_imported_field > 0", documents.get(0))); + } + + @Test + public void imported_fields_only_supported_for_simple_expressions() throws ParseException { + exceptionRule.expect(IllegalArgumentException.class); + // TODO we should probably handle this case specially and give a better exception message + exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test"); + + var documents = createDocs(); + // Nested field access is NOT considered a simple expression. + evaluate("test.my_imported_field.foo", documents.get(0)); + } + + @Test public void testTicket1769674() { assertParseError("music.uri=\"junk", "Lexical error at line -1, column 17. Encountered: <EOF> after : \"\\\"junk\""); diff --git a/document/src/test/resources/predicates/false__java b/document/src/test/resources/predicates/false__java Binary files differindex 00a71d5fe73..c9f426b3aaf 100644 --- a/document/src/test/resources/predicates/false__java +++ b/document/src/test/resources/predicates/false__java diff --git a/document/src/test/resources/predicates/foo_in_6_9__java b/document/src/test/resources/predicates/foo_in_6_9__java Binary files differindex d18b937bead..9c1413f1b77 100644 --- a/document/src/test/resources/predicates/foo_in_6_9__java +++ b/document/src/test/resources/predicates/foo_in_6_9__java diff --git a/document/src/test/resources/predicates/foo_in_6_x__java b/document/src/test/resources/predicates/foo_in_6_x__java Binary files differindex b8ae334882e..d53f4d03996 100644 --- a/document/src/test/resources/predicates/foo_in_6_x__java +++ b/document/src/test/resources/predicates/foo_in_6_x__java diff --git a/document/src/test/resources/predicates/foo_in_bar__java b/document/src/test/resources/predicates/foo_in_bar__java Binary files differindex aeb10e2b5d7..3adde3ae2fe 100644 --- a/document/src/test/resources/predicates/foo_in_bar__java +++ b/document/src/test/resources/predicates/foo_in_bar__java diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java Binary files differindex 8a2705e6f62..8583f37da57 100644 --- a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java +++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__java b/document/src/test/resources/predicates/foo_in_bar_baz__java Binary files differindex ea3314d9bd7..1db1a175c0c 100644 --- a/document/src/test/resources/predicates/foo_in_bar_baz__java +++ b/document/src/test/resources/predicates/foo_in_bar_baz__java diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java Binary files differindex 4d5474c24e4..3b1dbd541fa 100644 --- a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java +++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java diff --git a/document/src/test/resources/predicates/foo_in_x_9__java b/document/src/test/resources/predicates/foo_in_x_9__java Binary files differindex 017a610a7d5..9a2bfc9af89 100644 --- a/document/src/test/resources/predicates/foo_in_x_9__java +++ b/document/src/test/resources/predicates/foo_in_x_9__java diff --git a/document/src/test/resources/predicates/foo_in_x__java b/document/src/test/resources/predicates/foo_in_x__java Binary files differindex 6537cc6bdeb..8faf7762be7 100644 --- a/document/src/test/resources/predicates/foo_in_x__java +++ b/document/src/test/resources/predicates/foo_in_x__java diff --git a/document/src/test/resources/predicates/foo_in_x_x__java b/document/src/test/resources/predicates/foo_in_x_x__java Binary files differindex 5060718417a..bf01a265651 100644 --- a/document/src/test/resources/predicates/foo_in_x_x__java +++ b/document/src/test/resources/predicates/foo_in_x_x__java diff --git a/document/src/test/resources/predicates/not_foo_in_bar__java b/document/src/test/resources/predicates/not_foo_in_bar__java Binary files differindex b654de6d53e..a7cbacbaf35 100644 --- a/document/src/test/resources/predicates/not_foo_in_bar__java +++ b/document/src/test/resources/predicates/not_foo_in_bar__java diff --git a/document/src/test/resources/predicates/true__java b/document/src/test/resources/predicates/true__java Binary files differindex 2b5da7409d5..87356a5ed44 100644 --- a/document/src/test/resources/predicates/true__java +++ b/document/src/test/resources/predicates/true__java diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java Binary files differindex 2c15c152558..cf878f0e689 100644 --- a/document/src/test/resources/tensor/empty_tensor__java +++ b/document/src/test/resources/tensor/empty_tensor__java diff --git a/document/src/test/resources/tensor/multi_cell_tensor__java b/document/src/test/resources/tensor/multi_cell_tensor__java Binary files differindex d923fc10559..deb53463fb5 100644 --- a/document/src/test/resources/tensor/multi_cell_tensor__java +++ b/document/src/test/resources/tensor/multi_cell_tensor__java diff --git a/document/src/test/resources/tensor/non_existing_tensor__java b/document/src/test/resources/tensor/non_existing_tensor__java Binary files differindex 08cbcac6dd3..7a1d95ff132 100644 --- a/document/src/test/resources/tensor/non_existing_tensor__java +++ b/document/src/test/resources/tensor/non_existing_tensor__java diff --git a/document/src/tests/data/serializejava-compressed.dat b/document/src/tests/data/serializejava-compressed.dat Binary files differindex 0ac391422ff..e11bb3b53db 100644 --- a/document/src/tests/data/serializejava-compressed.dat +++ b/document/src/tests/data/serializejava-compressed.dat diff --git a/document/src/tests/data/serializejava.dat b/document/src/tests/data/serializejava.dat Binary files differindex 10873c9d905..3fa21bdccd3 100644 --- a/document/src/tests/data/serializejava.dat +++ b/document/src/tests/data/serializejava.dat diff --git a/document/src/tests/data/serializejavawithannotations.dat b/document/src/tests/data/serializejavawithannotations.dat Binary files differindex fe683d1580e..08854d03a29 100644 --- a/document/src/tests/data/serializejavawithannotations.dat +++ b/document/src/tests/data/serializejavawithannotations.dat diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 79b849c5ba9..c1e5fbecd14 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -16,6 +16,7 @@ #include <vespa/document/select/invalidconstant.h> #include <vespa/document/select/doctype.h> #include <vespa/document/select/compare.h> +#include <vespa/document/select/operator.h> #include <vespa/document/select/parse_utils.h> #include <vespa/vespalib/util/exceptions.h> #include <limits> @@ -56,16 +57,6 @@ protected: void SetUp() override; void createDocs(); - void testOperators0(); - void testOperators1(); - void testOperators2(); - void testOperators3(); - void testOperators4(); - void testOperators5(); - void testOperators6(); - void testOperators7(); - void testOperators8(); - void testOperators9(); void testDocumentUpdates0(); void testDocumentUpdates1(); void testDocumentUpdates2(); @@ -92,6 +83,10 @@ void DocumentSelectParserTest::SetUp() builder.document(-1673092522, "usergroup", Struct("usergroup.header"), Struct("usergroup.body")); + builder.document(1234567, "with_imported", + Struct("with_imported.header"), + Struct("with_imported.body")) + .imported_field("my_imported_field"); _repo = std::make_unique<DocumentTypeRepo>(builder.config()); _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); @@ -103,7 +98,7 @@ Document::SP DocumentSelectParserTest::createDoc( uint64_t hlong) { const DocumentType* type = _repo->getDocumentType(doctype); - Document::SP doc(new Document(*type, DocumentId(id))); + auto doc = std::make_shared<Document>(*type, DocumentId(id)); doc->setValue(doc->getField("headerval"), IntFieldValue(hint)); if (hlong != 0) { @@ -498,21 +493,7 @@ DocumentSelectParserTest::doParse(vespalib::stringref expr, EXPECT_EQ(select::ResultList(select::Result::result), \ doParse(expr, (doc).getId())) << (std::string("Doc id: ") + expr); -TEST_F(DocumentSelectParserTest, testOperators) -{ - testOperators0(); - testOperators1(); - testOperators2(); - testOperators3(); - testOperators4(); - testOperators5(); - testOperators6(); - testOperators7(); - testOperators8(); - testOperators9(); -} - -void DocumentSelectParserTest::testOperators0() +TEST_F(DocumentSelectParserTest, operators_0) { createDocs(); @@ -551,8 +532,17 @@ void DocumentSelectParserTest::testOperators0() PARSE("\"foo\" == 'foo'", *_doc[0], True); PARSE("\"bar\" = \"a\"", *_doc[0], False); PARSE("\"bar\" = \"*a*\"", *_doc[0], True); + PARSE("\"bar\" = \"*x*\"", *_doc[0], False); + PARSE("\"bar\" = \"ba*\"", *_doc[0], True); + PARSE("\"bar\" = \"a*\"", *_doc[0], False) + PARSE("\"bar\" = \"*ar\"", *_doc[0], True); + PARSE("\"bar\" = \"*a\"", *_doc[0], False); PARSE("\"bar\" = \"\"", *_doc[0], False); PARSE("\"\" = \"\"", *_doc[0], True); + PARSE("\"\" = \"*\"", *_doc[0], True); + PARSE("\"\" = \"****\"", *_doc[0], True); + PARSE("\"a\" = \"*?*\"", *_doc[0], True); + PARSE("\"a\" = \"*??*\"", *_doc[0], False); PARSE("\"bar\" =~ \"^a$\"", *_doc[0], False); PARSE("\"bar\" =~ \"a\"", *_doc[0], True); PARSE("\"bar\" =~ \"\"", *_doc[0], True); @@ -561,7 +551,16 @@ void DocumentSelectParserTest::testOperators0() PARSE("30 = 30", *_doc[0], True); } -void DocumentSelectParserTest::testOperators1() +TEST_F(DocumentSelectParserTest, regex_matching_does_not_bind_anchors_to_newlines) { + createDocs(); + + PARSE("\"a\\nb\\nc\" =~ \"^b$\"", *_doc[0], False); + PARSE("\"a\\r\\nb\\r\\nc\" =~ \"^b$\"", *_doc[0], False); + // Same applies to implicit regex created from glob expression + PARSE("\"a\\nb\\nc\" = \"b\"", *_doc[0], False); +} + +TEST_F(DocumentSelectParserTest, operators_1) { createDocs(); @@ -608,7 +607,7 @@ void DocumentSelectParserTest::testOperators1() PARSE("testdoctype1.headerval = 10", *_doc[4], True); } -void DocumentSelectParserTest::testOperators2() +TEST_F(DocumentSelectParserTest, operators_2) { createDocs(); @@ -633,7 +632,7 @@ void DocumentSelectParserTest::testOperators2() PARSEI("id.group == \"xyzzy\"", *_doc[10], True); } -void DocumentSelectParserTest::testOperators3() +TEST_F(DocumentSelectParserTest, operators_3) { createDocs(); { @@ -666,7 +665,7 @@ void DocumentSelectParserTest::testOperators3() PARSEI("id == \"id:footype:testdoctype1:n=123456789:badger\"", *_doc[5], False); } -void DocumentSelectParserTest::testOperators4() +TEST_F(DocumentSelectParserTest, operators_4) { createDocs(); @@ -693,7 +692,7 @@ void DocumentSelectParserTest::testOperators4() PARSE("false or testdoctype1.content = 1", *_doc[0], Invalid); } -void DocumentSelectParserTest::testOperators5() +TEST_F(DocumentSelectParserTest, operators_5) { createDocs(); @@ -722,7 +721,7 @@ void DocumentSelectParserTest::testOperators5() PARSEI("-6 % 10 = -6", *_doc[0], True); } -void DocumentSelectParserTest::testOperators6() +TEST_F(DocumentSelectParserTest, operators_6) { createDocs(); @@ -766,7 +765,7 @@ void DocumentSelectParserTest::testOperators6() PARSE("testdoctype1.headerlongval<0", *_doc[7], True); } -void DocumentSelectParserTest::testOperators7() +TEST_F(DocumentSelectParserTest, operators_7) { createDocs(); @@ -803,7 +802,7 @@ void DocumentSelectParserTest::testOperators7() PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$y].value == \"structval2\"", *_doc[1], True); } -void DocumentSelectParserTest::testOperators8() +TEST_F(DocumentSelectParserTest, operators_8) { createDocs(); @@ -836,7 +835,7 @@ void DocumentSelectParserTest::testOperators8() PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$y}[$x].value == \"structval2\"", *_doc[1], False); } -void DocumentSelectParserTest::testOperators9() +TEST_F(DocumentSelectParserTest, operators_9) { createDocs(); @@ -1047,6 +1046,11 @@ void DocumentSelectParserTest::testDocumentUpdates0() PARSEI("\"foo\" == \"foo\"", *_update[0], True); PARSEI("\"bar\" = \"a\"", *_update[0], False); PARSEI("\"bar\" = \"*a*\"", *_update[0], True); + PARSEI("\"bar\" = \"**\"", *_update[0], True); + PARSEI("\"bar\" = \"***\"", *_update[0], True); + PARSEI("\"bar\" = \"****\"", *_update[0], True); + PARSEI("\"bar\" = \"???\"", *_update[0], True); + PARSEI("\"bar\" = \"????\"", *_update[0], False); PARSEI("\"bar\" = \"\"", *_update[0], False); PARSEI("\"\" = \"\"", *_update[0], True); PARSEI("\"bar\" =~ \"^a$\"", *_update[0], False); @@ -1524,4 +1528,48 @@ TEST_F(DocumentSelectParserTest, test_parse_utilities_handle_malformed_input) check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity()); } +TEST_F(DocumentSelectParserTest, imported_field_references_are_treated_as_valid_field_with_missing_value) { + const DocumentType* type = _repo->getDocumentType("with_imported"); + ASSERT_TRUE(type != nullptr); + Document doc(*type, DocumentId("id::with_imported::foo")); + + PARSE("with_imported.my_imported_field == null", doc, True); + PARSE("with_imported.my_imported_field != null", doc, False); + PARSE("with_imported.my_imported_field", doc, False); + // Only (in)equality operators are well defined for null values; everything else becomes Invalid. + PARSE("with_imported.my_imported_field > 0", doc, Invalid); +} + +TEST_F(DocumentSelectParserTest, imported_field_references_only_support_for_simple_expressions) { + const DocumentType* type = _repo->getDocumentType("with_imported"); + ASSERT_TRUE(type != nullptr); + Document doc(*type, DocumentId("id::with_imported::foo")); + + PARSE("with_imported.my_imported_field.foo", doc, Invalid); + PARSE("with_imported.my_imported_field[0]", doc, Invalid); + PARSE("with_imported.my_imported_field{foo}", doc, Invalid); +} + +TEST_F(DocumentSelectParserTest, prefix_and_suffix_wildcard_globs_are_rewritten_to_optimized_form) { + using select::GlobOperator; + EXPECT_EQ(GlobOperator::convertToRegex("*foo"), "foo$"); + EXPECT_EQ(GlobOperator::convertToRegex("foo*"), "^foo"); + EXPECT_EQ(GlobOperator::convertToRegex("*foo*"), "foo"); + EXPECT_EQ(GlobOperator::convertToRegex("*"), ""); // Matches any string. + EXPECT_EQ(GlobOperator::convertToRegex("**"), ""); // Still matches any string. +} + +TEST_F(DocumentSelectParserTest, redundant_glob_wildcards_are_collapsed_into_minimal_form) { + using select::GlobOperator; + EXPECT_EQ(GlobOperator::convertToRegex("***"), ""); // Even still matches any string. + EXPECT_EQ(GlobOperator::convertToRegex("**foo**"), "foo"); + EXPECT_EQ(GlobOperator::convertToRegex("foo***"), "^foo"); + EXPECT_EQ(GlobOperator::convertToRegex("***foo"), "foo$"); + EXPECT_EQ(GlobOperator::convertToRegex("foo**bar"), "^foo.*bar$"); + EXPECT_EQ(GlobOperator::convertToRegex("**foo*bar**"), "foo.*bar"); + EXPECT_EQ(GlobOperator::convertToRegex("**foo***bar**"), "foo.*bar"); + EXPECT_EQ(GlobOperator::convertToRegex("*?*"), "."); + EXPECT_EQ(GlobOperator::convertToRegex("*?*?*?*"), "..*..*."); // Don't try this at home, kids! +} + } // document diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index fed4de67f28..968470e9693 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -585,7 +585,7 @@ TEST(DocumentTest, testReadSerializedFile) EXPECT_TRUE(buf2.empty()); buf2.rp(0); - EXPECT_EQ(len - 13, buf2.size()); // Size is smaller as we are merging to one chunk. + EXPECT_EQ(len, buf2.size()); doc2.setValue("stringfield", StringFieldValue("hei")); diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 5543cb48ba4..18001c35da5 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -872,6 +872,13 @@ struct TensorUpdateFixture { EXPECT_EQ(actTensor, expTensor); } + void assertTensorNull() { + auto field = getTensor(); + auto tensor_field = dynamic_cast<TensorFieldValue*>(field.get()); + ASSERT_TRUE(tensor_field); + EXPECT_TRUE(tensor_field->getAsTensorPtr().get() == nullptr); + } + void assertTensor(const TensorSpec &expSpec) { auto expTensor = makeTensor(expSpec); assertTensor(*expTensor); @@ -886,6 +893,19 @@ struct TensorUpdateFixture { assertTensor(expTensor); } + void assertApplyUpdateNonExisting(const ValueUpdate &update, + const TensorSpec &expTensor) { + applyUpdate(update); + assertDocumentUpdated(); + assertTensor(expTensor); + } + + void assertApplyUpdateNonExisting(const ValueUpdate &update) { + applyUpdate(update); + assertDocumentUpdated(); + assertTensorNull(); + } + template <typename ValueUpdateType> void assertRoundtripSerialize(const ValueUpdateType &valueUpdate) { testRoundtripSerialize(valueUpdate, tensorDataType); @@ -933,6 +953,16 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied) .add({{"x", "c"}}, 7)); } +TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorAddUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7))), + + f.spec().add({{"x", "b"}}, 5) + .add({{"x", "c"}}, 7)); +} + TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) { TensorUpdateFixture f; @@ -944,6 +974,12 @@ TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied) f.spec().add({{"x", "a"}}, 2)); } +TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorRemoveUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 1)))); +} + TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) { TensorUpdateFixture f; @@ -970,6 +1006,13 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied) .add({{"x", "b"}}, 15)); } +TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied_to_nonexisting_tensor) +{ + TensorUpdateFixture f; + f.assertApplyUpdateNonExisting(TensorModifyUpdate(TensorModifyUpdate::Operation::ADD, + f.makeTensor(f.spec().add({{"x", "b"}}, 5)))); +} + TEST(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized) { TensorUpdateFixture f; diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index b263ad75930..0bc80ebcd16 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -532,6 +532,47 @@ TEST("Reference fields are resolved to correct reference type") { EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType()); } +TEST("Config with no imported fields has empty imported fields set in DocumentType") { + DocumenttypesConfigBuilderHelper builder; + builder.document(doc_type_id, type_name, + Struct(header_name), Struct(body_name)); + DocumentTypeRepo repo(builder.config()); + const auto *type = repo.getDocumentType(doc_type_id); + ASSERT_TRUE(type != nullptr); + EXPECT_TRUE(type->imported_field_names().empty()); + EXPECT_FALSE(type->has_imported_field_name("foo")); +} + +TEST("Configured imported field names are available in the DocumentType") { + const int type_2_id = doc_type_id + 1; + // Note: we cheat a bit by specifying imported field names in types that have no + // reference fields. Add to test if we add config read-time validation of this. :) + DocumenttypesConfigBuilderHelper builder; + // Type with one imported field + builder.document(doc_type_id, type_name, + Struct(header_name), Struct(body_name)) + .imported_field("my_cool_field"); + // Type with two imported fields + builder.document(type_2_id, type_name_2, + Struct(header_name_2), Struct(body_name_2)) + .imported_field("my_awesome_field") + .imported_field("my_swag_field"); + + DocumentTypeRepo repo(builder.config()); + const auto* type = repo.getDocumentType(doc_type_id); + ASSERT_TRUE(type != nullptr); + EXPECT_EQUAL(1u, type->imported_field_names().size()); + EXPECT_TRUE(type->has_imported_field_name("my_cool_field")); + EXPECT_FALSE(type->has_imported_field_name("my_awesome_field")); + + type = repo.getDocumentType(type_2_id); + ASSERT_TRUE(type != nullptr); + EXPECT_EQUAL(2u, type->imported_field_names().size()); + EXPECT_TRUE(type->has_imported_field_name("my_awesome_field")); + EXPECT_TRUE(type->has_imported_field_name("my_swag_field")); + EXPECT_FALSE(type->has_imported_field_name("my_cool_field")); +} + namespace { const TensorDataType & diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 1961f67d83c..092a29d9293 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int ## Field sets datatype[].documenttype[].fieldsets{}.fields[] string +## Imported fields (specified outside the document block in the schema) +datatype[].documenttype[].importedfield[].name string + ## Cross-document reference with ID of target document type datatype[].referencetype[].target_type_id int diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index fb3fcd8b123..0f0a9e3e37c 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -104,3 +104,6 @@ documenttype[].referencetype[].id int ## Numeric ID of the document type instances of the reference point to. documenttype[].referencetype[].target_type_id int + +## Imported fields (specified outside the document block in the schema) +documenttype[].importedfield[].name string diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp index ffc250eacba..c2739dc1303 100644 --- a/document/src/vespa/document/datatype/documenttype.cpp +++ b/document/src/vespa/document/datatype/documenttype.cpp @@ -23,9 +23,10 @@ DocumentType::DocumentType() = default; DocumentType::DocumentType(stringref name, int32_t id) : StructuredDataType(name, id), _inheritedTypes(), - _ownedFields(new StructDataType(name + ".header")), + _ownedFields(std::make_shared<StructDataType>(name + ".header")), _fields(_ownedFields.get()), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { _inheritedTypes.push_back(DataType::DOCUMENT); @@ -36,7 +37,8 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie : StructuredDataType(name, id), _inheritedTypes(), _fields(&fields), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { _inheritedTypes.push_back(DataType::DOCUMENT); @@ -46,12 +48,13 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie DocumentType::DocumentType(stringref name) : StructuredDataType(name), _inheritedTypes(), - _ownedFields(new StructDataType(name + ".header")), + _ownedFields(std::make_shared<StructDataType>(name + ".header")), _fields(_ownedFields.get()), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { - _inheritedTypes.push_back(DataType::DOCUMENT); + _inheritedTypes.emplace_back(DataType::DOCUMENT); } } @@ -59,27 +62,28 @@ DocumentType::DocumentType(stringref name, const StructDataType& fields) : StructuredDataType(name), _inheritedTypes(), _fields(&fields), - _fieldSets() + _fieldSets(), + _imported_field_names() { if (name != "document") { - _inheritedTypes.push_back(DataType::DOCUMENT); + _inheritedTypes.emplace_back(DataType::DOCUMENT); } } DocumentType::~DocumentType() = default; DocumentType & -DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields) +DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields) { - _fieldSets[name] = FieldSet(name, fields); + _fieldSets[name] = FieldSet(name, std::move(fields)); return *this; } const DocumentType::FieldSet * DocumentType::getFieldSet(const vespalib::string & name) const { - FieldSetMap::const_iterator it(_fieldSets.find(name)); - return (it != _fieldSets.end()) ? & it->second : NULL; + auto it = _fieldSets.find(name); + return (it != _fieldSets.end()) ? & it->second : nullptr; } void @@ -107,37 +111,34 @@ DocumentType::inherit(const DocumentType &docType) { "Document type " + docType.toString() + " already inherits type " + toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC); } - // If we already inherits this type, there is no point in adding it - // again. + // If we already inherits this type, there is no point in adding it again. if (isA(docType)) { - // If we already directly inherits it, complain - for (std::vector<const DocumentType *>::const_iterator - it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it) - { - if (**it == docType) { + // If we already directly inherits it, complain + for (const auto* inherited : _inheritedTypes) { + if (*inherited == docType) { throw IllegalArgumentException( "DocumentType " + getName() + " already inherits " "document type " + docType.getName(), VESPA_STRLOC); } } - // Indirectly already inheriting it is oki, as this can happen - // due to inherited documents inheriting the same type. + // Indirectly already inheriting it is oki, as this can happen + // due to inherited documents inheriting the same type. LOG(info, "Document type %s inherits document type %s from multiple " "types.", getName().c_str(), docType.getName().c_str()); return; } - // Add non-conflicting types. + // Add non-conflicting types. Field::Set fs = docType._fields->getFieldSet(); - for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) { + for (const auto* field : fs) { if (!_ownedFields.get()) { _ownedFields.reset(_fields->clone()); _fields = _ownedFields.get(); } - _ownedFields->addInheritedField(**it); + _ownedFields->addInheritedField(*field); } // If we inherit default document type Document.0, remove that if adding // another parent, as that has to also inherit Document - if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) { + if ((_inheritedTypes.size() == 1) && (*_inheritedTypes[0] == *DataType::DOCUMENT)) { _inheritedTypes.clear(); } _inheritedTypes.push_back(&docType); @@ -168,8 +169,7 @@ DocumentType::print(std::ostream& out, bool verbose, const std::string& indent) out << ")"; if (verbose) { if (!_inheritedTypes.empty()) { - std::vector<const DocumentType *>::const_iterator it( - _inheritedTypes.begin()); + auto it = _inheritedTypes.begin(); out << "\n" << indent << " : "; (*it)->print(out, false, ""); while (++it != _inheritedTypes.end()) { @@ -188,17 +188,18 @@ DocumentType::operator==(const DataType& other) const { if (&other == this) return true; if (!DataType::operator==(other)) return false; - const DocumentType* o(dynamic_cast<const DocumentType*>(&other)); - if (o == 0) return false; + const auto* o(dynamic_cast<const DocumentType*>(&other)); + if (o == nullptr) return false; if (*_fields != *o->_fields) return false; if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false; - std::vector<const DocumentType *>::const_iterator it1(_inheritedTypes.begin()); - std::vector<const DocumentType *>::const_iterator it2(o->_inheritedTypes.begin()); + auto it1 = _inheritedTypes.begin(); + auto it2 = o->_inheritedTypes.begin(); while (it1 != _inheritedTypes.end()) { if (**it1 != **it2) return false; ++it1; ++it2; } + // TODO imported fields? like in the Java impl, field sets are not considered either... :I return true; } @@ -228,6 +229,14 @@ DocumentType::getFieldSet() const return _fields->getFieldSet(); } +bool DocumentType::has_imported_field_name(const vespalib::string& name) const noexcept { + return (_imported_field_names.find(name) != _imported_field_names.end()); +} + +void DocumentType::add_imported_field_name(const vespalib::string& name) { + _imported_field_names.insert(name); +} + DocumentType * DocumentType::clone() const { return new DocumentType(*this); diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h index 7fcf5dfa237..ed6e9e66ab5 100644 --- a/document/src/vespa/document/datatype/documenttype.h +++ b/document/src/vespa/document/datatype/documenttype.h @@ -12,9 +12,11 @@ #pragma once #include <vespa/document/datatype/structdatatype.h> - +#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> +#include <set> namespace document { @@ -25,12 +27,17 @@ class DocumentType : public StructuredDataType { public: class FieldSet { public: - typedef std::set<vespalib::string> Fields; - FieldSet() : _name(), _fields() {} - FieldSet(const vespalib::string & name) : _name(name), _fields() {} - FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {} - const vespalib::string & getName() const { return _name; } - const Fields & getFields() const { return _fields; } + using Fields = std::set<vespalib::string>; + FieldSet() = default; + explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {} + FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {} + FieldSet(const FieldSet&) = default; + FieldSet& operator=(const FieldSet&) = default; + FieldSet(FieldSet&&) noexcept = default; + FieldSet& operator=(FieldSet&&) noexcept = default; + + const vespalib::string & getName() const noexcept { return _name; } + const Fields & getFields() const noexcept { return _fields; } FieldSet & add(vespalib::string & field) { _fields.insert(field); return *this; @@ -39,28 +46,31 @@ public: vespalib::string _name; Fields _fields; }; - typedef std::map<vespalib::string, FieldSet> FieldSetMap; + using FieldSetMap = std::map<vespalib::string, FieldSet>; + using ImportedFieldNames = vespalib::hash_set<vespalib::string>; + std::vector<const DocumentType *> _inheritedTypes; - StructDataType::SP _ownedFields; - const StructDataType* _fields; - FieldSetMap _fieldSets; + StructDataType::SP _ownedFields; + const StructDataType* _fields; + FieldSetMap _fieldSets; + ImportedFieldNames _imported_field_names; public: - typedef std::unique_ptr<DocumentType> UP; - typedef std::shared_ptr<DocumentType> SP; + using UP = std::unique_ptr<DocumentType>; + using SP = std::shared_ptr<DocumentType>; DocumentType(); DocumentType(vespalib::stringref name, int32_t id); DocumentType(vespalib::stringref name, int32_t id, const StructDataType& fields); - DocumentType(vespalib::stringref name); + explicit DocumentType(vespalib::stringref name); DocumentType(vespalib::stringref name, const StructDataType& fields); - ~DocumentType(); + ~DocumentType() override; - const StructDataType& getFieldsType() const { return *_fields; } + const StructDataType& getFieldsType() const noexcept { return *_fields; } void addField(const Field&); @@ -89,9 +99,16 @@ public: Field::Set getFieldSet() const override; DocumentType* clone() const override; - DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields); + DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields); const FieldSet * getFieldSet(const vespalib::string & name) const; + const ImportedFieldNames& imported_field_names() const noexcept { + return _imported_field_names; + } + bool has_imported_field_name(const vespalib::string& name) const noexcept; + // Ideally the type would be immutable, but this is how it's built today. + void add_imported_field_name(const vespalib::string& name); + DECLARE_IDENTIFIABLE(DocumentType); }; diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp index 9b318a39f0a..56d7b6ab078 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp @@ -104,6 +104,19 @@ TensorFieldValue::operator=(std::unique_ptr<Tensor> rhs) } +void +TensorFieldValue::make_empty_if_not_existing() +{ + if (!_tensor) { + TensorSpec empty_spec(_dataType.getTensorType().to_spec()); + auto empty_value = Engine::ref().from_spec(empty_spec); + auto tensor_ptr = dynamic_cast<Tensor*>(empty_value.get()); + assert(tensor_ptr != nullptr); + _tensor.reset(tensor_ptr); + empty_value.release(); + } +} + void TensorFieldValue::accept(FieldValueVisitor &visitor) diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h index 80b18be55a0..ea3f8dea9be 100644 --- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h @@ -28,6 +28,8 @@ public: TensorFieldValue &operator=(const TensorFieldValue &rhs); TensorFieldValue &operator=(std::unique_ptr<vespalib::tensor::Tensor> rhs); + void make_empty_if_not_existing(); + virtual void accept(FieldValueVisitor &visitor) override; virtual void accept(ConstFieldValueVisitor &visitor) const override; virtual const DataType *getDataType() const override; diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h index 15ee0da0c79..52ed8d878a2 100644 --- a/document/src/vespa/document/repo/configbuilder.h +++ b/document/src/vespa/document/repo/configbuilder.h @@ -38,9 +38,9 @@ struct TypeOrId { }; struct Struct : DatatypeConfig { - Struct(const vespalib::string &name) { + explicit Struct(vespalib::string name) { type = Type::STRUCT; - sstruct.name = name; + sstruct.name = std::move(name); } Struct &setCompression(Sstruct::Compression::Type t, int32_t level, int32_t threshold, int32_t min_size) { @@ -63,7 +63,7 @@ struct Struct : DatatypeConfig { }; struct Array : DatatypeConfig { - Array(TypeOrId nested_type) { + explicit Array(TypeOrId nested_type) { addNestedType(nested_type); type = Type::ARRAY; array.element.id = nested_type.id; @@ -71,7 +71,7 @@ struct Array : DatatypeConfig { }; struct Wset : DatatypeConfig { - Wset(TypeOrId nested_type) { + explicit Wset(TypeOrId nested_type) { addNestedType(nested_type); type = Type::WSET; wset.key.id = nested_type.id; @@ -94,7 +94,7 @@ struct Map : DatatypeConfig { }; struct AnnotationRef : DatatypeConfig { - AnnotationRef(int32_t annotation_type_id) { + explicit AnnotationRef(int32_t annotation_type_id) { type = Type::ANNOTATIONREF; annotationref.annotation.id = annotation_type_id; } @@ -103,7 +103,7 @@ struct AnnotationRef : DatatypeConfig { struct DocTypeRep { DocumenttypesConfig::Documenttype &doc_type; - DocTypeRep(DocumenttypesConfig::Documenttype &type) + explicit DocTypeRep(DocumenttypesConfig::Documenttype &type) : doc_type(type) {} DocTypeRep &inherit(int32_t id) { doc_type.inherits.resize(doc_type.inherits.size() + 1); @@ -127,6 +127,12 @@ struct DocTypeRep { doc_type.referencetype.back().targetTypeId = target_type_id; return *this; } + + DocTypeRep& imported_field(vespalib::string field_name) { + doc_type.importedfield.resize(doc_type.importedfield.size() + 1); + doc_type.importedfield.back().name = std::move(field_name); + return *this; + } }; class DocumenttypesConfigBuilderHelper { diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index da59f527115..58dbd16beb6 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -441,10 +441,10 @@ void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, Do for (const auto & entry : fsv) { const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second); DocumentType::FieldSet::Fields fields; - for (size_t j(0); j < fs.fields.size(); j++) { - fields.insert(fs.fields[j]); + for (const auto& f : fs.fields) { + fields.insert(f); } - doc_type.addFieldSet(entry.first, fields); + doc_type.addFieldSet(entry.first, std::move(fields)); } } @@ -457,6 +457,14 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference } } +void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldVector& imported_fields, + DocumentType& doc_type) +{ + for (const auto& imported : imported_fields) { + doc_type.add_imported_field_name(imported.name); + } +} + void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) { DataTypeRepo *data_types = type_map[doc_type.id]; inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations); @@ -467,6 +475,7 @@ void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, Do setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo); inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type); addFieldSet(doc_type.fieldsets, *data_types->doc_type); + add_imported_fields(doc_type.importedfield, *data_types->doc_type); } void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) { diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index eaa795549bf..ef2ee26bdbd 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -191,35 +191,64 @@ GlobOperator::traceImpl(const Value& a, const Value& b, std::ostream& ost) const return match(left->getValue(), regex); } +namespace { + +// Returns the number of consecutive wildcard ('*') characters found from +// _and including_ the character at `i`, i.e. the wildcard run length. +size_t wildcard_run_length(size_t i, vespalib::stringref str) { + size_t n = 0; + for (; (i < str.size()) && (str[i] == '*'); ++n, ++i) {} + return n; +} + +} + vespalib::string -GlobOperator::convertToRegex(vespalib::stringref globpattern) const +GlobOperator::convertToRegex(vespalib::stringref globpattern) { + if (globpattern.empty()) { + return "^$"; // Empty glob can only match the empty string. + } vespalib::asciistream ost; - ost << '^'; - for(uint32_t i=0, n=globpattern.size(); i<n; ++i) { + size_t i = 0; + if (globpattern[0] != '*') { + ost << '^'; + } else { + i += wildcard_run_length(0, globpattern); // Skip entire prefix wildcard run. + } + const size_t n = globpattern.size(); + for (; i < n; ++i) { switch(globpattern[i]) { - case '*': ost << ".*"; - break; - case '?': ost << "."; - break; - case '^': - case '$': - case '|': - case '{': - case '}': - case '(': - case ')': - case '[': - case ']': - case '\\': - case '+': - case '.': ost << '\\' << globpattern[i]; - break; - // Are there other regex special chars we need to escape? - default: ost << globpattern[i]; + case '*': + i += wildcard_run_length(i, globpattern) - 1; // -1 since we always inc by 1 anyway. + if (i != (n - 1)) { // Don't emit trailing wildcard. + ost << ".*"; + } + break; + case '?': + ost << '.'; + break; + case '^': + case '$': + case '|': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case '\\': + case '+': + case '.': + ost << '\\' << globpattern[i]; + break; + // Are there other regex special chars we need to escape? + default: ost << globpattern[i]; } } - ost << '$'; + if (globpattern[n - 1] != '*') { + ost << '$'; + } return ost.str(); } diff --git a/document/src/vespa/document/select/operator.h b/document/src/vespa/document/select/operator.h index 7383b6cf545..ad73d284c63 100644 --- a/document/src/vespa/document/select/operator.h +++ b/document/src/vespa/document/select/operator.h @@ -88,7 +88,27 @@ public: // Delegates to Value::globCompare ResultList compare(const Value& a, const Value& b) const override; ResultList trace(const Value&, const Value&, std::ostream& trace) const override; - vespalib::string convertToRegex(vespalib::stringref globpattern) const; + /** + * Converts a standard glob expression into a regular expression string, + * supporting the following glob semantics: + * '*' matches 0-n arbitrary characters + * '?' matches exactly 1 arbitrary character + * This code simplifies the resulting regex as much as possible to help + * minimize the number of possible catastrophic backtracking cases that + * can be triggered by wildcard regexes. + * + * The following simplifications are currently performed: + * - '' -> /^$/ (empty string match) + * '*' -> // (any string match) + * - '*foo*' -> /foo/ (substring match) + * - '*foo' -> /foo$/ (suffix match) + * - 'foo*' -> /^foo/ (prefix match) + * - collapsing runs of consecutive '*' wildcards into a single + * wildcard. *** is identical to ** which is identical to * etc, + * as all these match 0-n characters each. This also works with + * simplification, i.e. '***foo***' -> /foo/ and '***' -> // + */ + static vespalib::string convertToRegex(vespalib::stringref globpattern); static bool containsVariables(vespalib::stringref expression); static const GlobOperator GLOB; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 5c7820b76d7..95cf2f4e7e5 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -340,10 +340,34 @@ FieldValueNode::initFieldPath(const DocumentType& type) const { } } +namespace { + +bool looks_like_complex_field_path(const vespalib::string& expr) { + for (const char c : expr) { + switch (c) { + case '.': + case '[': + case '{': + return true; + default: continue; + } + } + return false; +} + +bool is_simple_imported_field(const vespalib::string& expr, const DocumentType& doc_type) { + if (looks_like_complex_field_path(expr)) { + return false; + } + return (doc_type.has_imported_field_name(expr)); +} + +} + std::unique_ptr<Value> FieldValueNode::getValue(const Context& context) const { - if (context._doc == NULL) { + if (context._doc == nullptr) { return std::make_unique<InvalidValue>(); } @@ -352,7 +376,17 @@ FieldValueNode::getValue(const Context& context) const if (!documentTypeEqualsName(doc.getType(), _doctype)) { return std::make_unique<InvalidValue>(); } - try{ + // Imported fields can only be meaningfully evaluated inside Proton, so we + // explicitly treat them as if they are valid fields with missing values. This + // will be treated the same as if it's a normal field by the selection operators. + // This avoids any awkward interaction with Invalid values or having to + // augment the FieldPath code with knowledge of imported fields. + // When a selection is running inside Proton, it will patch FieldValueNodes for + // imported fields, which removes this check entirely. + if (is_simple_imported_field(_fieldExpression, doc.getType())) { + return std::make_unique<NullValue>(); + } + try { initFieldPath(doc.getType()); IteratorHandler handler; @@ -363,7 +397,7 @@ FieldValueNode::getValue(const Context& context) const } else { const std::vector<ArrayValue::VariableValue>& values = handler.getValues(); - if (values.size() == 0) { + if (values.empty()) { return std::make_unique<NullValue>(); } else { return std::make_unique<ArrayValue>(handler.getValues()); @@ -399,7 +433,7 @@ FieldValueNode::print(std::ostream& out, bool verbose, std::unique_ptr<Value> FieldValueNode::traceValue(const Context &context, std::ostream& out) const { - if (context._doc == NULL) { + if (context._doc == nullptr) { return defaultTrace(getValue(context), out); } const Document &doc(*context._doc); @@ -408,7 +442,12 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const << _doctype << " document, thus resolving invalid.\n"; return std::make_unique<InvalidValue>(); } - try{ + if (is_simple_imported_field(_fieldExpression, doc.getType())) { + out << "Field '" << _fieldExpression << "' refers to an imported field; " + << "returning NullValue to treat this as an unset field value.\n"; + return std::make_unique<NullValue>(); + } + try { initFieldPath(doc.getType()); IteratorHandler handler; diff --git a/document/src/vespa/document/update/tensor_add_update.cpp b/document/src/vespa/document/update/tensor_add_update.cpp index c35a8058133..2e5fa194c20 100644 --- a/document/src/vespa/document/update/tensor_add_update.cpp +++ b/document/src/vespa/document/update/tensor_add_update.cpp @@ -93,6 +93,7 @@ TensorAddUpdate::applyTo(FieldValue& value) const { if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); + tensorFieldValue.make_empty_if_not_existing(); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); auto newTensor = applyTo(*oldTensor); if (newTensor) { diff --git a/document/src/vespa/document/update/tensor_modify_update.cpp b/document/src/vespa/document/update/tensor_modify_update.cpp index 0d821de8922..dfc7479e5cd 100644 --- a/document/src/vespa/document/update/tensor_modify_update.cpp +++ b/document/src/vespa/document/update/tensor_modify_update.cpp @@ -174,9 +174,11 @@ TensorModifyUpdate::applyTo(FieldValue& value) const if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); - auto newTensor = applyTo(*oldTensor); - if (newTensor) { - tensorFieldValue = std::move(newTensor); + if (oldTensor) { + auto newTensor = applyTo(*oldTensor); + if (newTensor) { + tensorFieldValue = std::move(newTensor); + } } } else { vespalib::string err = make_string("Unable to perform a tensor modify update on a '%s' field value", diff --git a/document/src/vespa/document/update/tensor_remove_update.cpp b/document/src/vespa/document/update/tensor_remove_update.cpp index f3bf7da7a0b..91b4c0a6ca3 100644 --- a/document/src/vespa/document/update/tensor_remove_update.cpp +++ b/document/src/vespa/document/update/tensor_remove_update.cpp @@ -120,9 +120,11 @@ TensorRemoveUpdate::applyTo(FieldValue &value) const if (value.inherits(TensorFieldValue::classId)) { TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value); auto &oldTensor = tensorFieldValue.getAsTensorPtr(); - auto newTensor = applyTo(*oldTensor); - if (newTensor) { - tensorFieldValue = std::move(newTensor); + if (oldTensor) { + auto newTensor = applyTo(*oldTensor); + if (newTensor) { + tensorFieldValue = std::move(newTensor); + } } } else { vespalib::string err = make_string("Unable to perform a tensor remove update on a '%s' field value", diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 36609c04219..0b924451907 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -78,9 +78,9 @@ void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_ auto info = fixture.find_all<DenseXWProductFunction>(); ASSERT_EQUAL(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); - EXPECT_EQUAL(info[0]->vectorSize(), vec_size); - EXPECT_EQUAL(info[0]->resultSize(), res_size); - EXPECT_EQUAL(info[0]->matrixHasCommonDimensionInnermost(), happy); + EXPECT_EQUAL(info[0]->vector_size(), vec_size); + EXPECT_EQUAL(info[0]->result_size(), res_size); + EXPECT_EQUAL(info[0]->common_inner(), happy); } vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common, diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index e578b28da18..1e17c8284cb 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -83,6 +83,19 @@ std::vector<Value::CREF> get_refs(const std::vector<Value::UP> &values) { } // namespace vespalib::eval::test +void +EvalFixture::detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const +{ + for (size_t i = 0; i < _function->num_params(); ++i) { + auto pos = param_repo.map.find(_function->param_name(i)); + ASSERT_TRUE(pos != param_repo.map.end()); + bool allow_tampering = allow_mutable && pos->second.is_mutable; + if (!allow_tampering) { + ASSERT_EQUAL(pos->second.value, _engine.to_spec(*_param_values[i])); + } + } +} + EvalFixture::EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo ¶m_repo, @@ -104,6 +117,7 @@ EvalFixture::EvalFixture(const TensorEngine &engine, { auto result_type = ValueType::from_spec(_result.type()); ASSERT_TRUE(!result_type.is_error()); + TEST_DO(detect_param_tampering(param_repo, allow_mutable)); } const TensorSpec diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 48f6a7e5d2e..1d39dc52cba 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -67,6 +67,8 @@ private: } } + void detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const; + public: EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo ¶m_repo, bool optimized = true, bool allow_mutable = false); diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 8bcaddba3b4..f2c374ea02d 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -7,6 +7,8 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/tensor/tensor.h> +#include <cblas.h> + namespace vespalib::tensor { using eval::ValueType; @@ -19,32 +21,29 @@ using namespace eval::operation; namespace { template <typename LCT, typename RCT> -struct HWSupport { - static double call(hwaccelrated::IAccelrated *, const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs) { - double result = 0.0; - for (size_t i = 0; i < lhs.size(); ++i) { - result += (lhs[i] * rhs[i]); - } - return result; +void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + double result = 0.0; + const LCT *lhs = lhs_cells.cbegin(); + const RCT *rhs = rhs_cells.cbegin(); + for (size_t i = 0; i < lhs_cells.size(); ++i) { + result += ((*lhs++) * (*rhs++)); } -}; -template <> struct HWSupport<float, float> { - static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<float> &lhs, const ConstArrayRef<float> &rhs) { - return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size()); - } -}; -template <> struct HWSupport<double, double> { - static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<double> &lhs, const ConstArrayRef<double> &rhs) { - return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size()); - } -}; + state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); +} -template <typename LCT, typename RCT> -void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t param) { - auto *hw = (hwaccelrated::IAccelrated *)(param); - auto lhs = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto rhs = DenseTensorView::typify_cells<RCT>(state.peek(0)); - double result = HWSupport<LCT,RCT>::call(hw, lhs, rhs); +void my_cblas_double_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + double result = cblas_ddot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); + state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); +} + +void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) { + auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1)); + auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + double result = cblas_sdot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1); state.pop_pop_push(state.stash.create<eval::DoubleValue>(result)); } @@ -53,21 +52,31 @@ struct MyDotProductOp { static auto get_fun() { return my_dot_product_op<LCT,RCT>; } }; +eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { + if (lct == rct) { + if (lct == ValueType::CellType::DOUBLE) { + return my_cblas_double_dot_product_op; + } + if (lct == ValueType::CellType::FLOAT) { + return my_cblas_float_dot_product_op; + } + } + return select_2<MyDotProductOp>(lct, rct); +} + } // namespace vespalib::tensor::<unnamed> DenseDotProductFunction::DenseDotProductFunction(const eval::TensorFunction &lhs_in, const eval::TensorFunction &rhs_in) - : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in), - _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator()) + : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in) { } eval::InterpretedFunction::Instruction DenseDotProductFunction::compile_self(Stash &) const { - auto op = select_2<MyDotProductOp>(lhs().result_type().cell_type(), - rhs().result_type().cell_type()); - return eval::InterpretedFunction::Instruction(op, (uint64_t)(_hwAccelerator.get())); + auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type()); + return eval::InterpretedFunction::Instruction(op); } bool diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h index d6181d33887..1d8f749689b 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/eval/eval/tensor_function.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace vespalib::tensor { @@ -13,7 +12,6 @@ namespace vespalib::tensor { class DenseDotProductFunction : public eval::tensor_function::Op2 { private: - hwaccelrated::IAccelrated::UP _hwAccelerator; using ValueType = eval::ValueType; public: DenseDotProductFunction(const eval::TensorFunction &lhs_in, diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp index ce0d4230c23..272ceb319c8 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -10,7 +10,7 @@ #include <vespa/eval/tensor/tensor.h> #include <assert.h> -#include <openblas/cblas.h> +#include <cblas.h> namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index 2db5b4e8f92..7b03f45e9cc 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -10,6 +10,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <assert.h> +#include <cblas.h> + namespace vespalib::tensor { using eval::ValueType; @@ -21,76 +23,59 @@ using namespace eval::operation; namespace { -template <typename LCT, typename RCT> -struct HWSupport { - static double call(hwaccelrated::IAccelrated *, const LCT *lhs, const RCT *rhs, size_t len) { - double result = 0.0; - for (size_t i = 0; i < len; ++i) { - result += (lhs[i] * rhs[i]); - } - return result; +template <typename LCT, typename RCT, bool common_inner> +double my_dot_product(const LCT *lhs, const RCT *rhs, size_t vector_size, size_t result_size) { + double result = 0.0; + for (size_t i = 0; i < vector_size; ++i) { + result += ((*lhs) * (*rhs)); + ++lhs; + rhs += (common_inner ? 1 : result_size); } -}; -template <> struct HWSupport<float, float> { - static double call(hwaccelrated::IAccelrated *hw, const float *lhs, const float *rhs, size_t len) { - return hw->dotProduct(lhs, rhs, len); - } -}; -template <> struct HWSupport<double, double> { - static double call(hwaccelrated::IAccelrated *hw, const double *lhs, const double *rhs, size_t len) { - return hw->dotProduct(lhs, rhs, len); - } -}; - -template <typename LCT, typename RCT, typename OCT> -void multiDotProduct(const DenseXWProductFunction::Self &self, - const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result) -{ - OCT *out = result.begin(); - const RCT *matrixP = matrixCells.cbegin(); - const LCT * const vectorP = vectorCells.cbegin(); - for (size_t row = 0; row < self._resultSize; ++row) { - double cell = HWSupport<LCT,RCT>::call(self._hwAccelerator.get(), vectorP, matrixP, self._vectorSize); - *out++ = cell; - matrixP += self._vectorSize; - } - assert(out == result.end()); - assert(matrixP == matrixCells.cend()); -} - -template <typename LCT, typename RCT, typename OCT> -void transposedProduct(const DenseXWProductFunction::Self &self, - const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result) -{ - OCT *out = result.begin(); - const RCT * const matrixP = matrixCells.cbegin(); - const LCT * const vectorP = vectorCells.cbegin(); - for (size_t row = 0; row < self._resultSize; ++row) { - double cell = 0; - for (size_t col = 0; col < self._vectorSize; ++col) { - cell += matrixP[col*self._resultSize + row] * vectorP[col]; - } - *out++ = cell; - } - assert(out == result.end()); + return result; } -template <typename LCT, typename RCT, bool commonDimensionInnermost> +template <typename LCT, typename RCT, bool common_inner> void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { - DenseXWProductFunction::Self *self = (DenseXWProductFunction::Self *)(param); - + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type; - auto vectorCells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto matrixCells = DenseTensorView::typify_cells<RCT>(state.peek(0)); - auto outputCells = state.stash.create_array<OCT>(self->_resultSize); - - if (commonDimensionInnermost) { - multiDotProduct(*self, vectorCells, matrixCells, outputCells); - } else { - transposedProduct(*self, vectorCells, matrixCells, outputCells); + auto vector_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); + auto dst_cells = state.stash.create_array<OCT>(self.result_size); + OCT *dst = dst_cells.begin(); + const RCT *matrix = matrix_cells.cbegin(); + for (size_t i = 0; i < self.result_size; ++i) { + *dst++ = my_dot_product<LCT,RCT,common_inner>(vector_cells.cbegin(), matrix, self.vector_size, self.result_size); + matrix += (common_inner ? self.vector_size : 1); } + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} + +template <bool common_inner> +void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); + auto vector_cells = DenseTensorView::typify_cells<double>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<double>(state.peek(0)); + auto dst_cells = state.stash.create_array<double>(self.result_size); + cblas_dgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, + common_inner ? self.result_size : self.vector_size, + common_inner ? self.vector_size : self.result_size, + 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1, + 0.0, dst_cells.begin(), 1); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); +} - state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, TypedCells(outputCells))); +template <bool common_inner> +void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) { + const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param)); + auto vector_cells = DenseTensorView::typify_cells<float>(state.peek(1)); + auto matrix_cells = DenseTensorView::typify_cells<float>(state.peek(0)); + auto dst_cells = state.stash.create_array<float>(self.result_size); + cblas_sgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans, + common_inner ? self.result_size : self.vector_size, + common_inner ? self.vector_size : self.result_size, + 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1, + 0.0, dst_cells.begin(), 1); + state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } template <bool common_inner> @@ -99,11 +84,24 @@ struct MyXWProductOp { static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; } }; -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_innermost) { - if (common_innermost) { - return select_2<MyXWProductOp<true> >(lct, rct); +template <bool common_inner> +eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) { + if (lct == rct) { + if (lct == ValueType::CellType::DOUBLE) { + return my_cblas_double_xw_product_op<common_inner>; + } + if (lct == ValueType::CellType::FLOAT) { + return my_cblas_float_xw_product_op<common_inner>; + } + } + return select_2<MyXWProductOp<common_inner>>(lct, rct); +} + +eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) { + if (common_inner) { + return my_select2<true>(lct, rct); } else { - return select_2<MyXWProductOp<false> >(lct, rct); + return my_select2<false>(lct, rct); } } @@ -129,42 +127,43 @@ bool isDenseXWProduct(const ValueType &res, const ValueType &vec, const ValueTyp } const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFunction &vec, const TensorFunction &mat, Stash &stash) { - bool common_is_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1); + bool common_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1); return stash.create<DenseXWProductFunction>(res, vec, mat, vec.result_type().dimensions()[0].size, res.dimensions()[0].size, - common_is_inner); + common_inner); } } // namespace vespalib::tensor::<unnamed> -DenseXWProductFunction::Self::Self(const eval::ValueType &resultType, - size_t vectorSize, - size_t resultSize) - : _resultType(resultType), - _vectorSize(vectorSize), - _resultSize(resultSize), - _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator()) -{} +DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in, + size_t vector_size_in, size_t result_size_in) + : result_type(result_type_in), + vector_size(vector_size_in), + result_size(result_size_in) +{ +} +DenseXWProductFunction::Self::~Self() = default; -DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &resultType, +DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &result_type, const eval::TensorFunction &vector_in, const eval::TensorFunction &matrix_in, - size_t vectorSize, - size_t resultSize, - bool matrixHasCommonDimensionInnermost) - : eval::tensor_function::Op2(resultType, vector_in, matrix_in), - _vectorSize(vectorSize), - _resultSize(resultSize), - _commonDimensionInnermost(matrixHasCommonDimensionInnermost) -{} + size_t vector_size, + size_t result_size, + bool common_inner) + : eval::tensor_function::Op2(result_type, vector_in, matrix_in), + _vector_size(vector_size), + _result_size(result_size), + _common_inner(common_inner) +{ +} eval::InterpretedFunction::Instruction DenseXWProductFunction::compile_self(Stash &stash) const { - Self &self = stash.create<Self>(result_type(), _vectorSize, _resultSize); + Self &self = stash.create<Self>(result_type(), _vector_size, _result_size); auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _commonDimensionInnermost); + rhs().result_type().cell_type(), _common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } @@ -172,9 +171,9 @@ void DenseXWProductFunction::visit_self(vespalib::ObjectVisitor &visitor) const { Super::visit_self(visitor); - visitor.visitInt("vector_size", _vectorSize); - visitor.visitInt("result_size", _resultSize); - visitor.visitBool("common_dimension_innermost", _commonDimensionInnermost); + visitor.visitInt("vector_size", _vector_size); + visitor.visitInt("result_size", _result_size); + visitor.visitBool("common_inner", _common_inner); } const TensorFunction & diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h index f2f4d67c0f0..d7c39fa45a2 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h @@ -4,7 +4,6 @@ #include <vespa/eval/eval/tensor_function.h> #include "dense_tensor_view.h" -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace vespalib::tensor { @@ -16,37 +15,34 @@ class DenseXWProductFunction : public eval::tensor_function::Op2 using Super = eval::tensor_function::Op2; public: struct Self { - const eval::ValueType _resultType; - const size_t _vectorSize; - const size_t _resultSize; - hwaccelrated::IAccelrated::UP _hwAccelerator; - Self(const eval::ValueType &resultType, - size_t vectorSize, - size_t resultSize); - ~Self() {} + eval::ValueType result_type; + size_t vector_size; + size_t result_size; + Self(const eval::ValueType &result_type_in, + size_t vector_size_in, size_t result_size_in); + ~Self(); }; private: - const size_t _vectorSize; - const size_t _resultSize; - bool _commonDimensionInnermost; + size_t _vector_size; + size_t _result_size; + bool _common_inner; public: - DenseXWProductFunction(const eval::ValueType &resultType, + DenseXWProductFunction(const eval::ValueType &result_type, const eval::TensorFunction &vector_in, const eval::TensorFunction &matrix_in, - size_t vectorSize, - size_t resultSize, - bool matrixHasCommonDimensionInnermost); + size_t vector_size, + size_t result_size, + bool common_inner); ~DenseXWProductFunction() {} bool result_is_mutable() const override { return true; } - size_t vectorSize() const { return _vectorSize; } - size_t resultSize() const { return _resultSize; } - - bool matrixHasCommonDimensionInnermost() const { return _commonDimensionInnermost; } + size_t vector_size() const { return _vector_size; } + size_t result_size() const { return _result_size; } + bool common_inner() const { return _common_inner; } eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override; void visit_self(vespalib::ObjectVisitor &visitor) const override; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5edab66e6b3..6380ae3cfbd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -114,6 +114,21 @@ public class Flags { "scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).", "Takes effect on next run of NodeRebooter"); + public static final UnboundBooleanFlag ENABLE_LARGE_ORCHESTRATOR_LOCKS = defineFeatureFlag( + "enable-large-orchestrator-locks", true, + "If enabled, the orchestrator will accumulate application locks during probe in batch suspension, " + + "and release them in reverse order only after the non-probe is complete. Can be set depending on " + + "parent hostname.", + "Takes immediate effect for new batch suspensions.", + HOSTNAME); + + public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag( + "retire-with-permanently-down", false, + "If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " + + "instead of ALLOWED_TO_BE_DOWN (old behavior).", + "Takes effect on the next run of RetiredExpirer.", + HOSTNAME); + public static final UnboundBooleanFlag ENABLE_DYNAMIC_PROVISIONING = defineFeatureFlag( "enable-dynamic-provisioning", false, "Provision a new docker host when we otherwise can't allocate a docker node", @@ -186,17 +201,11 @@ public class Flags { APPLICATION_ID); public static final UnboundBooleanFlag USE_CONFIG_SERVER_FOR_TESTER_API_CALLS = defineFeatureFlag( - "use-config-server-for-tester-api-calls", false, + "use-config-server-for-tester-api-calls", true, "Whether controller should send requests to tester API through config server (if false) or tester endpoint (if true)", "Takes effect immediately", ZONE_ID); - public static final UnboundBooleanFlag INSTALL_L4_ROUTING_SUPPORT = defineFeatureFlag( - "install-l4-routing-support", false, - "Whether routing nodes should install package supporting L4 routing", - "Takes effect immediately", - ZONE_ID, HOSTNAME); - public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( "generate-l4-routing-config", false, "Whether routing nodes should generate L4 routing config", @@ -204,7 +213,7 @@ public class Flags { ZONE_ID, HOSTNAME); public static final UnboundBooleanFlag GENERATE_ROUTING_CONFIG_FOR_TESTER_APPLICATIONS = defineFeatureFlag( - "generate-routing-config-for-tester-applications", true, + "generate-routing-config-for-tester-applications", false, "Whether config server should generate routing config (lb-services) for tester applications", "Takes effect on the next deployment of the routing (zone) application", ZONE_ID); @@ -215,6 +224,17 @@ public class Flags { "Takes effect on the next deployment of the application", APPLICATION_ID); + public static final UnboundBooleanFlag USE_NEW_ATHENZ_FILTER = defineFeatureFlag( + "use-new-athenz-filter", false, + "Use new Athenz filter that supports access-tokens", + "Takes effect at redeployment", + APPLICATION_ID); + + public static final UnboundBooleanFlag USE_RPM_PACKAGES_FOR_DATA_HIGHWAY = defineFeatureFlag( + "use-rpm-packages-for-data-highway", false, + "Whether RPM packages should be used for Data Highway", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 74a7d19387c..5f7adb32af0 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -13,6 +13,8 @@ #include <vespa/log/log.h> LOG_SETUP(".fnet"); +std::atomic<uint64_t> FNET_Connection::_num_connections = 0; + namespace { class SyncPacket : public FNET_DummyPacket { private: @@ -489,6 +491,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _cleanup(nullptr) { assert(_socket && (_socket->get_fd() >= 0)); + _num_connections.fetch_add(1, std::memory_order_relaxed); } @@ -526,6 +529,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _adminChannel = admin.get(); _channels.Register(admin.release()); } + _num_connections.fetch_add(1, std::memory_order_relaxed); } @@ -536,6 +540,7 @@ FNET_Connection::~FNET_Connection() delete _adminChannel; } assert(_cleanup == nullptr); + _num_connections.fetch_sub(1, std::memory_order_relaxed); } diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 760b1b96d4d..c9c49c5151a 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -10,6 +10,7 @@ #include <vespa/vespalib/net/socket_handle.h> #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/crypto_socket.h> +#include <atomic> class FNET_IPacketStreamer; class FNET_IServerAdapter; @@ -111,6 +112,8 @@ private: FNET_IConnectionCleanupHandler *_cleanup; // cleanup handler + static std::atomic<uint64_t> _num_connections; // total number of connections + FNET_Connection(const FNET_Connection &); FNET_Connection &operator=(const FNET_Connection &); @@ -517,5 +520,10 @@ public: */ uint32_t getInputBufferSize() const { return _input.GetBufSize(); } + /** + * @return the total number of connection objects + **/ + static uint64_t get_num_connections() { + return _num_connections.load(std::memory_order_relaxed); + } }; - diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 1615cf7e686..508a6a84974 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -119,9 +119,16 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.util.Collection)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()" ], - "fields": [] + "fields": [ + "public java.util.List enabledCipherSuites", + "public java.util.List enabledProtocols" + ] }, "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum": { "superClass": "java.lang.Enum", @@ -174,7 +181,11 @@ "public java.lang.String certificate()", "public java.lang.String caCertificateFile()", "public java.lang.String caCertificate()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()" + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()", + "public java.util.List enabledCipherSuites()", + "public java.lang.String enabledCipherSuites(int)", + "public java.util.List enabledProtocols()", + "public java.lang.String enabledProtocols(int)" ], "fields": [] }, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index ffff63a424e..2d54c716f8d 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -8,19 +8,23 @@ import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.impl.NoConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.ssl.SSLContexts; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.security.GeneralSecurityException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,6 +57,9 @@ class HealthCheckProxyHandler extends HandlerWrapper { ConnectorConfig.HealthCheckProxy proxyConfig = connector.connectorConfig().healthCheckProxy(); if (proxyConfig.enable()) { mapping.put(connector.listenPort(), createProxyTarget(proxyConfig.port(), connectors)); + log.info(String.format("Port %1$d is configured as a health check proxy for port %2$d. " + + "HTTP requests to '%3$s' on %1$d are proxied as HTTPS to %2$d.", + connector.listenPort(), proxyConfig.port(), HEALTH_CHECK_PATH)); } } return mapping; @@ -63,9 +70,9 @@ class HealthCheckProxyHandler extends HandlerWrapper { .filter(connector -> connector.listenPort() == targetPort) .findAny() .orElseThrow(() -> new IllegalArgumentException("Could not find any connector with listen port " + targetPort)); - SslContextFactory sslContextFactory = + SslContextFactory.Server sslContextFactory = Optional.ofNullable(targetConnector.getConnectionFactory(SslConnectionFactory.class)) - .map(SslConnectionFactory::getSslContextFactory) + .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); return new ProxyTarget(targetPort, sslContextFactory); } @@ -111,10 +118,10 @@ class HealthCheckProxyHandler extends HandlerWrapper { private static class ProxyTarget implements AutoCloseable { final int port; - final SslContextFactory sslContextFactory; + final SslContextFactory.Server sslContextFactory; volatile CloseableHttpClient client; - ProxyTarget(int port, SslContextFactory sslContextFactory) { + ProxyTarget(int port, SslContextFactory.Server sslContextFactory) { this.port = port; this.sslContextFactory = sslContextFactory; } @@ -133,7 +140,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { client = HttpClientBuilder.create() .disableAutomaticRetries() .setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE) - .setSSLContext(sslContextFactory.getSslContext()) + .setSSLContext(getSslContext(sslContextFactory)) .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 .setUserAgent("health-check-proxy-client") @@ -144,6 +151,25 @@ class HealthCheckProxyHandler extends HandlerWrapper { return client; } + private SSLContext getSslContext(SslContextFactory.Server sslContextFactory) { + if (sslContextFactory.getNeedClientAuth()) { + log.info(String.format("Port %d requires client certificate. HTTPS client will use the target server connector's ssl context.", port)); + // A client certificate is only required if the server connector's ssl context factory is configured with "need-auth". + // We use the server's ssl context (truststore + keystore) if a client certificate is required. + // This will only work if the server certificate's CA is in the truststore. + return sslContextFactory.getSslContext(); + } else { + log.info(String.format( + "Port %d does not require a client certificate. HTTPS client will use a custom ssl context accepting all certificates.", port)); + // No client certificate required. The client is configured with a trust manager that accepts all certificates. + try { + return SSLContexts.custom().loadTrustMaterial(new TrustAllStrategy()).build(); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + } + @Override public void close() throws IOException { synchronized (this) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java index 48a7c246500..90848f1dfd4 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -2,14 +2,16 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.security.tls.DefaultTlsContext; -import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.AutoReloadingX509KeyManager; import com.yahoo.security.tls.TlsContext; import org.eclipse.jetty.util.ssl.SslContextFactory; +import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; @@ -17,16 +19,21 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.security.PrivateKey; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; + /** * An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. * * @author bjorncs */ -public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider { +public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider { + private volatile AutoReloadingX509KeyManager keyManager; private final ConnectorConfig connectorConfig; public ConfiguredSslContextFactoryProvider(ConnectorConfig connectorConfig) { @@ -35,17 +42,50 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider } @Override - protected TlsContext getTlsContext(String containerId, int port) { + public SslContextFactory getInstance(String containerId, int port) { ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); if (!sslConfig.enabled()) throw new IllegalStateException(); - PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig)); - List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig)); + SslContextBuilder builder = new SslContextBuilder(); + if (sslConfig.certificateFile().isBlank() || sslConfig.privateKeyFile().isBlank()) { + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig)); + List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig)); + builder.withKeyStore(privateKey, certificates); + } else { + keyManager = AutoReloadingX509KeyManager.fromPemFiles(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile())); + builder.withKeyManager(keyManager); + } List<X509Certificate> caCertificates = getCaCertificates(sslConfig) .map(X509CertificateUtils::certificateListFromPem) .orElse(List.of()); - PeerAuthentication peerAuthentication = toPeerAuthentication(sslConfig.clientAuth()); - return new DefaultTlsContext(certificates, privateKey, caCertificates, null, null, peerAuthentication); + builder.withTrustStore(caCertificates); + + SSLContext sslContext = builder.build(); + + SslContextFactory.Server factory = new SslContextFactory.Server(); + factory.setSslContext(sslContext); + + factory.setNeedClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.NEED_AUTH); + factory.setWantClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.WANT_AUTH); + + List<String> protocols = !sslConfig.enabledProtocols().isEmpty() + ? sslConfig.enabledProtocols() + : new ArrayList<>(TlsContext.getAllowedProtocols(sslContext)); + setEnabledProtocols(factory, sslContext, protocols); + + List<String> ciphers = !sslConfig.enabledCipherSuites().isEmpty() + ? sslConfig.enabledCipherSuites() + : new ArrayList<>(TlsContext.getAllowedCipherSuites(sslContext)); + setEnabledCipherSuites(factory, sslContext, ciphers); + + return factory; + } + + @Override + public void close() { + if (keyManager != null) { + keyManager.close(); + } } private static void validateConfig(ConnectorConfig.Ssl config) { @@ -64,19 +104,6 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider throw new IllegalArgumentException("Specified neither private key or private key file."); } - private static PeerAuthentication toPeerAuthentication(ConnectorConfig.Ssl.ClientAuth.Enum clientAuth) { - switch (clientAuth) { - case DISABLED: - return PeerAuthentication.DISABLED; - case NEED_AUTH: - return PeerAuthentication.NEED; - case WANT_AUTH: - return PeerAuthentication.WANT; - default: - throw new IllegalArgumentException("Unknown client auth: " + clientAuth); - } - } - private static boolean hasBoth(String a, String b) { return !a.isBlank() && !b.isBlank(); } private static boolean hasNeither(String a, String b) { return a.isBlank() && b.isBlank(); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java new file mode 100644 index 00000000000..a0172668cbb --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLContext; +import java.util.Arrays; +import java.util.List; + +/** + * @author bjorncs + */ +class SslContextFactoryUtils { + + static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContext, List<String> enabledCiphers) { + String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); + factory.setIncludeCipherSuites(enabledCiphers.toArray(String[]::new)); + factory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); + } + + static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContext, List<String> enabledProtocols) { + String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); + factory.setIncludeProtocols(enabledProtocols.toArray(String[]::new)); + factory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + } + + private static String[] createExclusionList(List<String> enabledValues, String[] supportedValues) { + return Arrays.stream(supportedValues) + .filter(supportedValue -> !enabledValues.contains(supportedValue)) + .toArray(String[]::new); + } +} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java index e8ae13e48be..93d4f1dca3f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java @@ -8,7 +8,10 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; -import java.util.Arrays; +import java.util.List; + +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; +import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; /** * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances. @@ -31,24 +34,9 @@ public abstract class TlsContextBasedProvider extends AbstractComponent implemen sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth()); sslContextFactory.setWantClientAuth(parameters.getWantClientAuth()); - String[] enabledProtocols = parameters.getProtocols(); - sslContextFactory.setIncludeProtocols(enabledProtocols); - String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); - sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); + setEnabledProtocols(sslContextFactory, sslContext, List.of(parameters.getProtocols())); + setEnabledCipherSuites(sslContextFactory, sslContext, List.of(parameters.getCipherSuites())); - String[] enabledCiphers = parameters.getCipherSuites(); - String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); - sslContextFactory.setIncludeCipherSuites(enabledCiphers); - sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); return sslContextFactory; } - - private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) { - return Arrays.stream(supportedValues) - .filter(supportedValue -> - Arrays.stream(enabledValues) - .noneMatch(enabledValue -> enabledValue.equals(supportedValue))) - .toArray(String[]::new); - } - } diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 1122b1db3a9..fe79ec2ffa3 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -81,6 +81,12 @@ ssl.caCertificate string default="" # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED +# List of enabled cipher suites. JDisc will use Vespa default if empty. +ssl.enabledCipherSuites[] string + +# List of enabled TLS protocol versions. JDisc will use Vespa default if empty. +ssl.enabledProtocols[] string + # Enforce TLS client authentication for https requests at the http layer. # Intended to be used with connectors with optional client authentication enabled. # 401 status code is returned for requests from non-authenticated clients. diff --git a/jrt/src/com/yahoo/jrt/Connector.java b/jrt/src/com/yahoo/jrt/Connector.java index 4c83a2884bd..57fad5a163d 100644 --- a/jrt/src/com/yahoo/jrt/Connector.java +++ b/jrt/src/com/yahoo/jrt/Connector.java @@ -1,75 +1,43 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; +import com.yahoo.concurrent.ThreadFactoryFactory; -class Connector { +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; - private class Run implements Runnable { - public void run() { - try { - Connector.this.run(); - } catch (Throwable problem) { - parent.handleFailure(problem, Connector.this); - } - } - } - - private Thread thread = new Thread(new Run(), "<jrt-connector>"); - private Transport parent; - private ThreadQueue connectQueue = new ThreadQueue(); - private boolean done = false; - private boolean exit = false; +class Connector { - public Connector(Transport parent) { - this.parent = parent; - thread.setDaemon(true); - thread.start(); - } + private final ExecutorService executor = new ThreadPoolExecutor(1, 64, 1L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getDaemonThreadFactory("jrt.connector")); - public void connectLater(Connection c) { - if ( ! connectQueue.enqueue(c)) { - c.transportThread().addConnection(c); - } + private void connect(Connection conn) { + conn.transportThread().addConnection(conn.connect()); } - private void run() { + public void connectLater(Connection conn) { try { - while (true) { - Connection conn = (Connection) connectQueue.dequeue(); - conn.transportThread().addConnection(conn.connect()); - } - } catch (EndOfQueueException e) {} - synchronized (this) { - done = true; - notifyAll(); - while (!exit) { - try { wait(); } catch (InterruptedException x) {} - } + executor.execute(() -> connect(conn)); + } catch (RejectedExecutionException e) { + conn.transportThread().addConnection(conn); } } public Connector shutdown() { - connectQueue.close(); - return this; - } - - public synchronized void waitDone() { - while (!done) { - try { wait(); } catch (InterruptedException x) {} - } - } - - public synchronized Connector exit() { - exit = true; - notifyAll(); + executor.shutdown(); return this; } public void join() { while (true) { try { - thread.join(); - return; + if (executor.awaitTermination(60, TimeUnit.SECONDS)) { + return; + } } catch (InterruptedException e) {} } } diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index f4eb1acd096..ad42409c48a 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -46,7 +46,7 @@ public class Transport { this.fatalHandler = fatalHandler; // NB: this must be set first } this.cryptoEngine = cryptoEngine; - connector = new Connector(this); + connector = new Connector(); worker = new Worker(this); runCnt = new AtomicInteger(numThreads); for (int i = 0; i < numThreads; ++i) { @@ -162,7 +162,7 @@ public class Transport { * @return this object, to enable chaining with join **/ public Transport shutdown() { - connector.shutdown().waitDone(); + connector.shutdown().join(); for (TransportThread thread: threads) { thread.shutdown(); } @@ -181,7 +181,6 @@ public class Transport { void notifyDone(TransportThread self) { if (runCnt.decrementAndGet() == 0) { worker.shutdown().join(); - connector.exit().join(); try { cryptoEngine.close(); } catch (Exception e) {} } } diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index 5a75c8b31ea..d465464de7f 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -8,6 +8,7 @@ "methods": [ "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)", "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, double)", + "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, java.lang.String)", "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(com.yahoo.tensor.Tensor)", "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(double)", "public com.yahoo.tensor.Tensor evaluate()", diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java index 994f6dd9b64..e373a54bcd1 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -2,6 +2,7 @@ package ai.vespa.models.evaluation; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -61,6 +62,21 @@ public class FunctionEvaluator { } /** + * Binds the given variable referred in this expression to the given value. + * String values are not yet supported in tensors. + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public FunctionEvaluator bind(String name, String value) { + if (evaluated) + throw new IllegalStateException("Cannot bind a new value in a used evaluator"); + context.put(name, new StringValue(value)); + return this; + } + + /** * Sets the default value to use for variables which are not bound * * @param value the default value diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index 5c353fcdf35..de23a8c6526 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -77,8 +77,14 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { property(request, missingValueKey).ifPresent(missingValue -> evaluator.setMissingValue(Tensor.from(missingValue))); for (Map.Entry<String, TensorType> argument : evaluator.function().argumentTypes().entrySet()) { - property(request, argument.getKey()).ifPresent(value -> evaluator.bind(argument.getKey(), - Tensor.from(argument.getValue(), value))); + Optional<String> value = property(request, argument.getKey()); + if (value.isPresent()) { + try { + evaluator.bind(argument.getKey(), Tensor.from(argument.getValue(), value.get())); + } catch (IllegalArgumentException e) { + evaluator.bind(argument.getKey(), value.get()); // since we don't yet support tensors with string values + } + } } Tensor result = evaluator.evaluate(); return new Response(200, JsonFormat.encode(result)); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java index 9320ac3fad8..0d13b7d4660 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java @@ -25,7 +25,7 @@ public class MlModelsImportingTest { public void testImportingModels() { ModelTester tester = new ModelTester("src/test/resources/config/models/"); - assertEquals(4, tester.models().size()); + assertEquals(5, tester.models().size()); // TODO: When we get type information in Models, replace the evaluator.context().names() check below by that { @@ -47,7 +47,24 @@ public class MlModelsImportingTest { } { + Model lightgbm = tester.models().get("lightgbm_regression"); + // Function + assertEquals(1, lightgbm.functions().size()); + ExpressionFunction function = tester.assertFunction("lightgbm_regression", + "(optimized sum of condition trees of size 480 bytes)", + lightgbm); + assertEquals("tensor()", function.returnType().get().toString()); + assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", commaSeparated(function.arguments())); + function.arguments().forEach(arg -> assertEquals(TensorType.empty, function.argumentTypes().get(arg))); + + // Evaluator + FunctionEvaluator evaluator = lightgbm.evaluatorOf(); + assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", evaluator.context().names().stream().sorted().collect(Collectors.joining(", "))); + assertEquals(1.91300868202, evaluator.evaluate().sum().asDouble(), delta); + } + + { Model onnxMnistSoftmax = tester.models().get("mnist_softmax"); // Function diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 629c82f410a..c9e49d3be02 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -56,7 +56,7 @@ public class ModelsEvaluationHandlerTest { public void testListModels() { String url = "http://localhost/model-evaluation/v1"; String expected = - "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\"}"; + "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; assertResponse(url, 200, expected); } @@ -94,6 +94,39 @@ public class ModelsEvaluationHandlerTest { } @Test + public void testLightGBMEvaluationWithoutBindings() { + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testLightGBMEvaluationWithBindings() { + Map<String, String> properties = new HashMap<>(); + properties.put("numerical_1", "0.1"); + properties.put("numerical_2", "0.2"); + properties.put("categorical_1", "a"); + properties.put("categorical_2", "i"); + properties.put("non-existing-binding", "-1"); + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":2.054697758469921}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test + public void testLightGBMEvaluationWithMissingValue() { + Map<String, String> properties = new HashMap<>(); + properties.put("missing-value", "-1.0"); + properties.put("numerical_2", "0.5"); + properties.put("categorical_1", "b"); + properties.put("categorical_2", "j"); + properties.put("non-existing-binding", "-1"); + String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":2.0745534018208094}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test public void testMnistSoftmaxDetails() { String url = "http://localhost:8080/model-evaluation/v1/mnist_softmax"; String expected = "{\"model\":\"mnist_softmax\",\"functions\":[{\"function\":\"default.add\",\"info\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add\",\"eval\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add/eval\",\"arguments\":[{\"name\":\"Placeholder\",\"type\":\"tensor(d0[],d1[784])\"}]}]}"; diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg index c25c5ba555b..385115b7cd4 100644 --- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg +++ b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg @@ -26,3 +26,6 @@ rankprofile[3].fef.property[3].name "rankingExpression(serving_default.y).input. rankprofile[3].fef.property[3].value "tensor(d0[],d1[784])" rankprofile[3].fef.property[4].name "rankingExpression(serving_default.y).type" rankprofile[3].fef.property[4].value "tensor(d1[10])" +rankprofile[4].name "lightgbm_regression" +rankprofile[4].fef.property[0].name "rankingExpression(lightgbm_regression).rankingScript" +rankprofile[4].fef.property[0].value "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in ["k", "l", "m"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in ["d", "e"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in ["a", "b"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in ["k", "l", "m"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)" diff --git a/model-integration/src/main/config/model-integration.xml b/model-integration/src/main/config/model-integration.xml index 90ec7d7275e..34f5f0ce31a 100644 --- a/model-integration/src/main/config/model-integration.xml +++ b/model-integration/src/main/config/model-integration.xml @@ -1,11 +1,12 @@ <!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!-- Component which can import some ml model. This is included into the config server services.xml to enable it to translate - model pseudofeatures in ranking expressions during config mddel building. + model pseudo features in ranking expressions during config model building. It is provided as separate bundles instead of being included in the config model because some of these (TensorFlow) includes JNI code, and so can only exist in one instance in the server. --> <component id="ai.vespa.rankingexpression.importer.onnx.OnnxImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter" bundle="model-integration" /> +<component id="ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter" bundle="model-integration" /> <component id="ai.vespa.rankingexpression.importer.vespa.VespaImporter" bundle="model-integration" /> diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java new file mode 100644 index 00000000000..76caa652ad2 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java @@ -0,0 +1,54 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import ai.vespa.rankingexpression.importer.ImportedModel; +import ai.vespa.rankingexpression.importer.ModelImporter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; + +import java.io.File; +import java.io.IOException; + +/** + * Converts a LightGBM model into a ranking expression. + * + * @author lesters + */ +public class LightGBMImporter extends ModelImporter { + + @Override + public boolean canImport(String modelPath) { + File modelFile = new File(modelPath); + if ( ! modelFile.isFile()) return false; + return modelFile.toString().endsWith(".json") && probe(modelFile); + } + + /** + * Returns true if the give file looks like a LightGBM json file. + * Currently, we just check if the json has an element called "tree_info" + */ + private boolean probe(File modelFile) { + try { + return new ObjectMapper().readTree(modelFile).has("tree_info"); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read '" + modelFile + "'", e); + } + } + + @Override + public ImportedModel importModel(String modelName, String modelPath) { + try { + ImportedModel model = new ImportedModel(modelName, modelPath); + LightGBMParser parser = new LightGBMParser(modelPath); + RankingExpression expression = new RankingExpression(parser.toRankingExpression()); + model.expression(modelName, expression); + return model; + } catch (IOException e) { + throw new IllegalArgumentException("Could not import LightGBM model from '" + modelPath + "'", e); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse ranking expression resulting from '" + modelPath + "'", e); + } + } + +} diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java new file mode 100644 index 00000000000..dc76ed8cb6f --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java @@ -0,0 +1,67 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +/** + * @author lesters + */ +public class LightGBMNode { + + // split nodes + private int split_feature; + private String threshold; // double for numerical, string for categorical + private String decision_type; + private boolean default_left; + private String missing_type; + private int internal_count; + private LightGBMNode left_child; + private LightGBMNode right_child; + + // leaf nodes + private double leaf_value; + private int leaf_count; + + public int getSplit_feature() { + return split_feature; + } + + public String getThreshold() { + return threshold; + } + + public String getDecision_type() { + return decision_type; + } + + public boolean isDefault_left() { + return default_left; + } + + public String getMissing_type() { + return missing_type; + } + + public int getInternal_count() { + return internal_count; + } + + public LightGBMNode getLeft_child() { + return left_child; + } + + public LightGBMNode getRight_child() { + return right_child; + } + + public double getLeaf_value() { + return leaf_value; + } + + public int getLeaf_count() { + return leaf_count; + } + + public boolean isLeaf() { + return left_child == null && right_child == null; + } + +}
\ No newline at end of file diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java new file mode 100644 index 00000000000..996343674ce --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java @@ -0,0 +1,146 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** + * @author lesters + */ +class LightGBMParser { + + private final String objective; + private final List<LightGBMNode> nodes; + private final List<String> featureNames; + private final Map<Integer, List<String>> categoryValues; // pr feature index + + LightGBMParser(String filePath) throws JsonProcessingException, IOException { + ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + JsonNode root = mapper.readTree(new File(filePath)); + + objective = root.get("objective").asText("regression"); + featureNames = parseFeatureNames(root); + nodes = parseTrees(mapper, root); + categoryValues = parseCategoryValues(root); + } + + private List<String> parseFeatureNames(JsonNode root) { + List<String> features = new ArrayList<>(); + for (JsonNode name : root.get("feature_names")) { + features.add(name.textValue()); + } + return features; + } + + private List<LightGBMNode> parseTrees(ObjectMapper mapper, JsonNode root) throws JsonProcessingException { + List<LightGBMNode> nodes = new ArrayList<>(); + for (JsonNode treeNode : root.get("tree_info")) { + nodes.add(mapper.treeToValue(treeNode.get("tree_structure"), LightGBMNode.class)); + } + return nodes; + } + + private Map<Integer, List<String>> parseCategoryValues(JsonNode root) { + Map<Integer, List<String>> categoryValues = new HashMap<>(); + + // Since the JSON format does not explicitly tell which features are + // categorical, we traverse the decision tree looking for categorical + // decisions and use that to determine which categorical features. + Set<Integer> categoricalFeatures = new TreeSet<>(); + nodes.forEach(node -> findCategoricalFeatures(node, categoricalFeatures)); + + // Again, the LightGBM JSON format does not explicitly tell which + // categorical values map to each categorical feature. The assumption + // here is that the order they appear in the "pandas_categorical" + // structure is the same order as the "feature_names". + var pandasFeatureIterator = root.get("pandas_categorical").iterator(); + var categoricalFeatureIterator = categoricalFeatures.iterator(); + while (pandasFeatureIterator.hasNext() && categoricalFeatureIterator.hasNext()) { + List<String> values = new ArrayList<>(); + pandasFeatureIterator.next().forEach(value -> values.add(value.textValue())); + categoryValues.put(categoricalFeatureIterator.next(), values); + } + + return categoryValues; + } + + private void findCategoricalFeatures(LightGBMNode node, Set<Integer> categoricalFeatures) { + if (node == null || node.isLeaf()) { + return; + } + if (node.getDecision_type().equals("==")) { + categoricalFeatures.add(node.getSplit_feature()); + } + findCategoricalFeatures(node.getLeft_child(), categoricalFeatures); + findCategoricalFeatures(node.getRight_child(), categoricalFeatures); + } + + String toRankingExpression() { + return applyObjective(nodes.stream().map(this::nodeToRankingExpression).collect(Collectors.joining(" + \n"))); + } + + // See https://lightgbm.readthedocs.io/en/latest/Parameters.html#objective + private String applyObjective(String expression) { + if (objective.startsWith("binary") || objective.equals("cross_entropy")) { + return "sigmoid(" + expression + ")"; + } + if (objective.equals("poisson") || objective.equals("gamma") || objective.equals("tweedie")) { + return "exp(" + expression + ")"; + } + return expression; // else: use expression directly + } + + private String nodeToRankingExpression(LightGBMNode node) { + if (node.isLeaf()) { + return Double.toString(node.getLeaf_value()); + } else { + String condition; + String feature = featureNames.get(node.getSplit_feature()); + if (node.getDecision_type().equals("==")) { + String values = transformCategoryIndexesToValues(node); + if (node.isDefault_left()) { // means go left (true) when isNan + condition = "isNan(" + feature + ") || (" + feature + " in [ " + values + "])"; + } else { + condition = feature + " in [" + values + "]"; + } + } else { // assumption: all other decision types are <= + double value = Double.parseDouble(node.getThreshold()); + if (node.isDefault_left()) { + condition = "!(" + feature + " >= " + value + ")"; + } else { + condition = feature + " < " + value; + } + } + String left = nodeToRankingExpression(node.getLeft_child()); + String right = nodeToRankingExpression(node.getRight_child()); + return "if (" + condition + ", " + left + ", " + right + ")"; + } + } + + private String transformCategoryIndexesToValues(LightGBMNode node) { + return Arrays.stream(node.getThreshold().split("\\|\\|")) + .map(index -> "\"" + transformCategoryIndexToValue(node.getSplit_feature(), index) + "\"") + .collect(Collectors.joining(",")); + } + + private String transformCategoryIndexToValue(int featureIndex, String valueIndex) { + if ( ! categoryValues.containsKey(featureIndex) ) { + return valueIndex; // We don't have a pandas categorical lookup table + } + return categoryValues.get(featureIndex).get(Integer.parseInt(valueIndex)); + } + +}
\ No newline at end of file diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java new file mode 100644 index 00000000000..b29145ee21b --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java new file mode 100644 index 00000000000..d2ef4b984ff --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer; +import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestNode; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * @author lesters + */ +public class LightGBMImportEvaluationTestCase extends LightGBMTestBase { + + @Test + public void testRegression() { + RankingExpression expression = importModel("src/test/models/lightgbm/regression.json"); + ArrayContext context = new ArrayContext(expression, true, DoubleValue.NaN); + + assertEvaluation(1.91300868, expression, features(context)); + assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + + ExpressionOptimizer optimizer = new ExpressionOptimizer(); + optimizer.optimize(expression, (ContextIndex)context); + assertTrue(expression.getRoot() instanceof GBDTForestNode); + + assertEvaluation(1.91300868, expression, features(context)); + assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + } + + @Test + public void testClassification() { + RankingExpression expression = importModel("src/test/models/lightgbm/classification.json"); + ArrayContext context = new ArrayContext(expression, DoubleValue.NaN); + assertEvaluation(0.37464997, expression, features(context)); + assertEvaluation(0.37464997, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i")); + assertEvaluation(0.38730827, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j")); + assertEvaluation(0.5647872, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m")); + } + +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java new file mode 100644 index 00000000000..80c2ce68394 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.rankingexpression.importer.lightgbm; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; +import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; + +import static org.junit.Assert.assertEquals; + +/** + * @author lesters + */ +class LightGBMTestBase { + + RankingExpression importModel(String path) { + return new LightGBMImporter().importModel("lightgbm", path).expressions().get("lightgbm"); + } + + void assertEvaluation(double expected, RankingExpression expr, TestFeatures features) { + assertEquals(expected, expr.evaluate(features.context).asDouble(), 1e-6); + } + + TestFeatures features(ArrayContext context) { + return new TestFeatures(context.clone()); + } + + static class TestFeatures { + private final ArrayContext context; + TestFeatures(ArrayContext context) { + this.context = context; + } + TestFeatures add(String name, double value) { + context.put(name, value); + return this; + } + TestFeatures add(String name, String value) { + context.put(name, new StringValue(value)); + return this; + } + } + +} diff --git a/model-integration/src/test/models/lightgbm/classification.json b/model-integration/src/test/models/lightgbm/classification.json new file mode 100644 index 00000000000..1087446519d --- /dev/null +++ b/model-integration/src/test/models/lightgbm/classification.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "binary sigmoid:1", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 13080.099609375, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 8303.599609375, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.598248, + "internal_weight": 14841.2, + "internal_count": 59371, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.10149588882231209, + "leaf_weight": 8812.104370772839, + "leaf_count": 35252 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.05076009488472203, + "leaf_weight": 6029.137221112847, + "leaf_count": 24119 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.1075553310531564, + "leaf_weight": 10156.217760130763, + "leaf_count": 40629 + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 12144.5, + "threshold": 0.4932456977560694, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.07039230856418545, + "leaf_weight": 12362.572675153613, + "leaf_count": 49561 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 6445.509765625, + "threshold": 0.4026061210695467, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.691647, + "internal_weight": 12581.6, + "internal_count": 50439, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.016713933964828474, + "leaf_weight": 5157.183633238077, + "leaf_count": 20675 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.12881836794307533, + "leaf_weight": 7424.385220557451, + "leaf_count": 29764 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 11470.099609375, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.0837843210726433, + "leaf_weight": 9858.360527098179, + "leaf_count": 39612 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 8077.8701171875, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": -0.549408, + "internal_weight": 15039.7, + "internal_count": 60388, + "left_child": { + "leaf_index": 1, + "leaf_value": 0.035561394754096094, + "leaf_weight": 5955.117423638701, + "leaf_count": 23896 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.11424082565448186, + "leaf_weight": 9084.538012728095, + "leaf_count": 36492 + } + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 11022.599609375, + "threshold": 0.5135386524711826, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 5789.919921875, + "threshold": 0.6237474076885036, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.641438, + "internal_weight": 12881.9, + "internal_count": 51907, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.11613056205533928, + "leaf_weight": 8044.6355674266815, + "leaf_count": 32426 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.022313103333779363, + "leaf_weight": 4837.266924858093, + "leaf_count": 19481 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.06927713686880098, + "leaf_weight": 11923.512641906738, + "leaf_count": 48093 + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 2, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 9828.9501953125, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 100000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.07771712562582928, + "leaf_weight": 9804.427681803703, + "leaf_count": 39586 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 6332.2900390625, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": -0.51112, + "internal_weight": 14922.7, + "internal_count": 60414, + "left_child": { + "leaf_index": 1, + "leaf_value": 0.029062142260340918, + "leaf_weight": 5933.120021238923, + "leaf_count": 23922 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.10400033924773491, + "leaf_weight": 8989.602796778083, + "leaf_count": 36492 + } + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/model-integration/src/test/models/lightgbm/regression.json b/model-integration/src/test/models/lightgbm/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/model-integration/src/test/models/lightgbm/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py new file mode 100755 index 00000000000..ac00437d192 --- /dev/null +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py @@ -0,0 +1,54 @@ +#! /usr/bin/env python3 +# coding: utf-8 + +import json +import random + +import lightgbm as lgb +import numpy as np +import pandas as pd + + +def category_value(arr): + values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 } + return np.array([ 0.21 * values[i] for i in arr ]) + +# Create training set +num_examples = 100000 +missing_prob = 0.01 +features = pd.DataFrame({ + "numerical_1": np.random.random(num_examples), + "numerical_2": np.random.random(num_examples), + "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"), + "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"), + }) + +# randomly insert missing values +for i in range(int(num_examples * len(features.columns) * missing_prob)): + features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None + +# create targets (with 0.0 as default for missing values) +target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"]) +target = (target > 2.24) * 1.0 +lgb_train = lgb.Dataset(features, target) + +# Train model +params = { + 'objective': 'binary', + 'metric': 'binary_logloss', + 'num_leaves': 3, +} +model = lgb.train(params, lgb_train, num_boost_round=5) + +# Save model +with open("classification.json", "w") as f: + json.dump(model.dump_model(), f, indent=2) + +# Predict (for comparison with Vespa evaluation) +predict_features = pd.DataFrame({ + "numerical_1": pd.Series([ None, 0.1, None, 0.7]), + "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]), + "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"), + "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"), + }) +print(model.predict(predict_features)) diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py new file mode 100755 index 00000000000..3e74e38da35 --- /dev/null +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 +# coding: utf-8 + +import json +import random + +import lightgbm as lgb +import numpy as np +import pandas as pd + + +def category_value(arr): + values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 } + return np.array([ 0.21 * values[i] for i in arr ]) + +# Create training set +num_examples = 100000 +missing_prob = 0.01 +features = pd.DataFrame({ + "numerical_1": np.random.random(num_examples), + "numerical_2": np.random.random(num_examples), + "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"), + "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"), + }) + +# randomly insert missing values +for i in range(int(num_examples * len(features.columns) * missing_prob)): + features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None + +# create targets (with 0.0 as default for missing values) +target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"]) +lgb_train = lgb.Dataset(features, target) + +# Train model +params = { + 'objective': 'mse', + 'metric': {'l2', 'l1'}, + 'num_leaves': 3, +} +model = lgb.train(params, lgb_train, num_boost_round=2) + +# Save model +with open("regression.json", "w") as f: + json.dump(model.dump_model(), f, indent=2) + +# Predict (for comparison with Vespa evaluation) +predict_features = pd.DataFrame({ + "numerical_1": pd.Series([ None, 0.1, None, 0.7]), + "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]), + "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"), + "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"), + }) +print(model.predict(predict_features)) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 68178418e62..2f92ef8affe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.log.LogLevel; import com.yahoo.vespa.flags.DoubleFlag; @@ -26,6 +27,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintai import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -47,11 +51,10 @@ public class NodeAgentImpl implements NodeAgent { // This is used as a definition of 1 GB when comparing flavor specs in node-repo private static final long BYTES_IN_GB = 1_000_000_000L; - private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); + // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration + private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(30); - private final AtomicBoolean terminated = new AtomicBoolean(false); - private boolean hasResumedNode = false; - private boolean hasStartedServices = true; + private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName()); private final NodeAgentContextSupplier contextSupplier; private final NodeRepository nodeRepository; @@ -61,12 +64,19 @@ public class NodeAgentImpl implements NodeAgent { private final Optional<CredentialsMaintainer> credentialsMaintainer; private final Optional<AclMaintainer> aclMaintainer; private final Optional<HealthChecker> healthChecker; + private final Clock clock; + private final Duration warmUpDuration; private final DoubleFlag containerCpuCap; private Thread loopThread; private ContainerState containerState = UNKNOWN; private NodeSpec lastNode; + private final AtomicBoolean terminated = new AtomicBoolean(false); + private boolean hasResumedNode = false; + private boolean hasStartedServices = true; + private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty(); + private int numberOfUnhandledException = 0; private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -87,16 +97,18 @@ public class NodeAgentImpl implements NodeAgent { // Created in NodeAdminImpl - public NodeAgentImpl( - final NodeAgentContextSupplier contextSupplier, - final NodeRepository nodeRepository, - final Orchestrator orchestrator, - final DockerOperations dockerOperations, - final StorageMaintainer storageMaintainer, - final FlagSource flagSource, - final Optional<CredentialsMaintainer> credentialsMaintainer, - final Optional<AclMaintainer> aclMaintainer, - final Optional<HealthChecker> healthChecker) { + public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, + Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer, + FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock) { + this(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, credentialsMaintainer, + aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION); + } + + public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository, + Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer, + FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, Duration warmUpDuration) { this.contextSupplier = contextSupplier; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; @@ -105,6 +117,8 @@ public class NodeAgentImpl implements NodeAgent { this.credentialsMaintainer = credentialsMaintainer; this.aclMaintainer = aclMaintainer; this.healthChecker = healthChecker; + this.clock = clock; + this.warmUpDuration = warmUpDuration; this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource); } @@ -194,9 +208,11 @@ public class NodeAgentImpl implements NodeAgent { } } - private void startContainer(NodeAgentContext context) { + private Container startContainer(NodeAgentContext context) { ContainerData containerData = createContainerData(context); - dockerOperations.createContainer(context, containerData, getContainerResources(context)); + ContainerResources wantedResources = context.nodeType() != NodeType.tenant || warmUpDuration.isNegative() ? + getContainerResources(context) : getContainerResources(context).withUnlimitedCpus(); + dockerOperations.createContainer(context, containerData, wantedResources); dockerOperations.startContainer(context); currentRebootGeneration = context.node().wantedRebootGeneration(); @@ -204,6 +220,8 @@ public class NodeAgentImpl implements NodeAgent { hasStartedServices = true; // Automatically started with the container hasResumedNode = false; context.log(logger, "Container successfully started, new containerState is " + containerState); + return dockerOperations.getContainer(context).orElseThrow(() -> + new ConvergenceException("Did not find container that was just started")); } private Optional<Container> removeContainerIfNeededUpdateContainerState( @@ -250,6 +268,7 @@ public class NodeAgentImpl implements NodeAgent { if (containerState == ABSENT) return; try { hasStartedServices = hasResumedNode = false; + firstSuccessfulHealthCheckInstant = Optional.empty(); dockerOperations.stopServices(context); } catch (ContainerNotFoundException e) { containerState = ABSENT; @@ -333,9 +352,15 @@ public class NodeAgentImpl implements NodeAgent { } - private void updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) { + private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) { ContainerResources wantedContainerResources = getContainerResources(context); - if (wantedContainerResources.equalsCpu(existingContainer.resources)) return; + + if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant + .map(clock.instant().minus(warmUpDuration)::isBefore) + .orElse(true)) + return existingContainer; + + if (wantedContainerResources.equalsCpu(existingContainer.resources)) return existingContainer; context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s", wantedContainerResources.toStringCpu(), existingContainer.resources.toStringCpu()); @@ -343,6 +368,8 @@ public class NodeAgentImpl implements NodeAgent { // Only update CPU resources dockerOperations.updateContainer(context, wantedContainerResources.withMemoryBytes(existingContainer.resources.memoryBytes())); + return dockerOperations.getContainer(context).orElseThrow(() -> + new ConvergenceException("Did not find container that was just updated")); } private ContainerResources getContainerResources(NodeAgentContext context) { @@ -430,16 +457,25 @@ public class NodeAgentImpl implements NodeAgent { credentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context)); if (container.isEmpty()) { containerState = STARTING; - startContainer(context); + container = Optional.of(startContainer(context)); containerState = UNKNOWN; } else { - updateContainerIfNeeded(context, container.get()); + container = Optional.of(updateContainerIfNeeded(context, container.get())); } aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); startServicesIfNeeded(context); resumeNodeIfNeeded(context); - healthChecker.ifPresent(checker -> checker.verifyHealth(context)); + if (healthChecker.isPresent()) { + healthChecker.get().verifyHealth(context); + if (firstSuccessfulHealthCheckInstant.isEmpty()) + firstSuccessfulHealthCheckInstant = Optional.of(clock.instant()); + + Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration)); + if (!container.get().resources.equalsCpu(getContainerResources(context))) + throw new ConvergenceException("Refusing to resume until warm up period ends (" + + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); + } // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 653a6f7f091..ea41ab21b9d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -86,18 +86,19 @@ public class DockerTester implements AutoCloseable { .build(); nodeRepository.updateNodeRepositoryNode(hostSpec); + Clock clock = Clock.systemUTC(); + Metrics metrics = new Metrics(); FileSystem fileSystem = TestFileSystem.create(); DockerOperations dockerOperations = new DockerOperationsImpl(docker, terminal, ipAddresses, flagSource); - Metrics metrics = new Metrics(); NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl( contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, - Optional.empty(), Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); + Optional.empty(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1)); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).pathToContainerStorageFromFileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, - nodeAdmin, HOST_HOSTNAME, Clock.systemUTC()); + nodeAdmin, HOST_HOSTNAME, clock); loopThread = new Thread(() -> { nodeAdminStateUpdater.start(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 9e1f84f07a1..f65ad379a63 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Container; @@ -26,12 +27,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import org.junit.Test; import org.mockito.InOrder; +import java.time.Duration; +import java.time.Instant; import java.util.Optional; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -65,6 +70,7 @@ public class NodeAgentImplTest { private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final ManualClock clock = new ManualClock(Instant.now()); @Test @@ -641,6 +647,86 @@ public class NodeAgentImplTest { verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId())); } + @Test + public void initial_cpu_cap_test() { + NodeSpec.Builder specBuilder = nodeBuilder + .state(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); + + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); + + InOrder inOrder = inOrder(orchestrator, dockerOperations); + + ConvergenceException healthCheckException = new ConvergenceException("Not yet up"); + doThrow(healthCheckException).when(healthChecker).verifyHealth(any()); + for (int i = 0; i < 3; i++) { + try { + nodeAgent.doConverge(context); + fail("Expected to fail with health check exception"); + } catch (ConvergenceException e) { + assertEquals(healthCheckException, e); + } + clock.advance(Duration.ofSeconds(30)); + } + + doNothing().when(healthChecker).verifyHealth(any()); + try { + nodeAgent.doConverge(context); + fail("Expected to fail due to warm up period not yet done"); + } catch (ConvergenceException e) { + assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); + } + inOrder.verify(orchestrator, never()).resume(any()); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(dockerOperations, never()).updateContainer(any(), any()); + + + clock.advance(Duration.ofSeconds(31)); + nodeAgent.doConverge(context); + + inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16))); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(dockerOperations, never()).startContainer(any()); + inOrder.verify(orchestrator).resume(any(String.class)); + + // No changes + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator).resume(any(String.class)); + } + + @Test + public void resumes_normally_if_container_is_already_capped_on_start() { + NodeSpec.Builder specBuilder = nodeBuilder + .state(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(1).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + + when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); + + InOrder inOrder = inOrder(orchestrator, dockerOperations); + + nodeAgent.doConverge(context); + + nodeAgent.converge(context); + inOrder.verify(orchestrator, never()).suspend(any(String.class)); + inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any()); + inOrder.verify(dockerOperations, never()).removeContainer(any(), any()); + inOrder.verify(orchestrator).resume(any(String.class)); + } + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { NodeSpec.Builder nodeBuilder = new NodeSpec.Builder() .resources(resources) @@ -672,11 +758,28 @@ public class NodeAgentImplTest { } private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { + return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1)); + } + + private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) { mockGetContainer(dockerImage, isRunning); + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(2, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(dockerOperations).createContainer(any(), any(), any()); + + doAnswer(invoc -> { + NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class); + ContainerResources resources = invoc.getArgument(1, ContainerResources.class); + mockGetContainer(context.node().wantedDockerImage().get(), resources, true); + return null; + }).when(dockerOperations).updateContainer(any(), any()); return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer), - Optional.of(healthChecker)); + Optional.of(healthChecker), clock, warmUpDuration); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 970972a5fe6..ed6e7cc71ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -51,7 +51,7 @@ public class MetricsReporter extends Maintainer { Clock clock) { super(nodeRepository, interval); this.metric = metric; - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.serviceMonitor = serviceMonitor; this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier; this.clock = clock; @@ -129,17 +129,15 @@ public class MetricsReporter extends Maintainer { metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context); metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context); - orchestrator.apply(new HostName(node.hostname())) - .ifPresent(info -> { - int suspended = info.status().isSuspended() ? 1 : 0; - metric.set("suspended", suspended, context); - metric.set("allowedToBeDown", suspended, context); // remove summer 2020. - - info.suspendedSince().ifPresent(suspendedSince -> { - Duration duration = Duration.between(suspendedSince, clock.instant()); - metric.set("suspendedSeconds", duration.getSeconds(), context); - }); - }); + orchestrator.apply(new HostName(node.hostname())).ifPresent(info -> { + int suspended = info.status().isSuspended() ? 1 : 0; + metric.set("suspended", suspended, context); + metric.set("allowedToBeDown", suspended, context); // remove summer 2020. + long suspendedSeconds = info.suspendedSince() + .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds()) + .orElse(0L); + metric.set("suspendedSeconds", suspendedSeconds, context); + }); long numberOfServices; HostName hostName = new HostName(node.hostname()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index b80446b06da..ca53b215237 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -265,7 +265,7 @@ public class NodeFailer extends Maintainer { private boolean nodeSuspended(Node node) { try { - return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN; + return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended(); } catch (HostNameNotFoundException e) { // Treat it as not suspended return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java index af32530a156..65cbc17aff5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.node; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.time.Instant; import java.util.Arrays; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java index fd6094ae111..7885cec6b65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java @@ -71,6 +71,6 @@ public class Reports { return this; } - public Reports build() { return new Reports(Collections.unmodifiableMap(reportMap)); } + public Reports build() { return new Reports(reportMap); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index ae4c93621e5..66172521d4c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -6,7 +6,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index b6a276e5e57..9614c1aa5c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -20,7 +20,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java index 37dc8a8a1ad..6615dff24e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java index dfa79a4fd9a..e27cdcd1842 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java index 14c8c7fa18e..915b1acedaa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.node.OsVersion; import java.io.IOException; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java index 1839f7187fa..1149b15a2b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java @@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.HashSet; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f0caa358cb8..1500154aa07 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -72,6 +72,7 @@ public class LoadBalancerProvisioner { public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type + if (application.instance().isTester()) return; // Do not provision for tester instances try (var loadBalancersLock = db.lockLoadBalancers()) { provision(application, cluster.id(), false, loadBalancersLock); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java index 09cb5dad0a9..11be95604c8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java @@ -6,12 +6,13 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -34,8 +35,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; -import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static com.yahoo.config.provision.NodeResources.StorageType.local; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; /** * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. @@ -46,7 +47,6 @@ import static com.yahoo.config.provision.NodeResources.StorageType.local; public class NodePatcher { private static final String WANT_TO_RETIRE = "wantToRetire"; - private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -100,7 +100,6 @@ public class NodePatcher { private List<Node> applyFieldRecursive(List<Node> childNodes, String name, Inspector value) { switch (name) { case WANT_TO_RETIRE: - case WANT_TO_DEPROVISION: return childNodes.stream() .map(child -> applyField(child, name, value)) .collect(Collectors.toList()); @@ -139,7 +138,9 @@ public class NodePatcher { return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes); case WANT_TO_RETIRE : return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case WANT_TO_DEPROVISION : + case "wantToDeprovision" : + if (node.type() != NodeType.host && asBoolean(value)) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); return node.with(node.status().withWantToDeprovision(asBoolean(value))); case "reports" : return nodeWithPatchedReports(node, value); @@ -172,22 +173,42 @@ public class NodePatcher { } private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { + Node patchedNode; // "reports": null clears the reports - if (reportsInspector.type() == Type.NIX) return node.with(new Reports()); + if (reportsInspector.type() == Type.NIX) { + patchedNode = node.with(new Reports()); + } else { + var reportsBuilder = new Reports.Builder(node.reports()); + reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { + if (reportInspector.type() == Type.NIX) { + // ... "reports": { "reportId": null } clears the report "reportId" + reportsBuilder.clearReport(reportId); + } else { + // ... "reports": { "reportId": {...} } overrides the whole report "reportId" + reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); + } + }); + patchedNode = node.with(reportsBuilder.build()); + } - var reportsBuilder = new Reports.Builder(node.reports()); + boolean hadHardFailReports = node.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); + boolean hasHardFailReports = patchedNode.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); - reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { - if (reportInspector.type() == Type.NIX) { - // ... "reports": { "reportId": null } clears the report "reportId" - reportsBuilder.clearReport(reportId); - } else { - // ... "reports": { "reportId": {...} } overrides the whole report "reportId" - reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); - } - }); + // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa + if (hadHardFailReports != hasHardFailReports) { + // Do not automatically change wantToDeprovision when + // 1. Transitioning to having a HARD_FAIL report and being in state failed: + // To allow operators manually unset before the host is parked and deleted. + // 2. When in parked state: Deletion is imminent, possibly already underway + if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) + return patchedNode; + + patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + } - return node.with(reportsBuilder.build()); + return patchedNode; } private Set<String> asStringSet(Inspector field) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index c64d1a9b5ff..f0d996ef595 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -21,7 +21,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.Type; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -51,7 +51,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; -import static com.yahoo.vespa.config.SlimeUtils.optionalString; +import static com.yahoo.slime.SlimeUtils.optionalString; /** * The implementation of the /nodes/v2 API. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 8a2156b0ce3..8ca8dfc26f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -16,12 +16,10 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; -import com.yahoo.vespa.orchestrator.status.HostStatus; import java.io.IOException; import java.io.OutputStream; @@ -59,7 +57,7 @@ class NodesResponse extends HttpResponse { this.nodeParentUrl = toNodeParentUrl(request); filter = NodesApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); - this.orchestrator = orchestrator.getNodeStatuses(); + this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; slime = new Slime(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index 888b8835e49..49d0ba5cf70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -13,7 +13,8 @@ public class ContainerConfig { return "<container version='1.0'>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>20</maxthreads>\n" + - " </config> \n" + + " </config>\n" + + " <accesslog type='disabled'/>\n" + " <component id='com.yahoo.test.ManualClock'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + " <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java index 3e671f03adf..d183e62c96c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java @@ -12,8 +12,10 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -23,7 +25,7 @@ import java.util.function.Function; */ public class OrchestratorMock extends AbstractComponent implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -33,14 +35,13 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = suspendedHosts.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostInfo>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)) - .map(status -> status.isSuspended() ? HostInfo.createSuspended(status, Instant.EPOCH) - : HostInfo.createNoRemarks()); + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -53,7 +54,7 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override @@ -78,7 +79,7 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator } @Override - public void acquirePermissionToRemove(HostName hostName) {} + public void acquirePermissionToRemove(HostName hostName) { } @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 287842e56f0..672709a2f8f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -93,7 +93,7 @@ public class MetricsReporterTest { ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> + when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))) ); ServiceModel serviceModel = mock(ServiceModel.class); @@ -144,7 +144,7 @@ public class MetricsReporterTest { Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); - when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); + when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); ServiceModel serviceModel = mock(ServiceModel.class); when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 6dfb1aed47e..c26614c630c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -218,9 +218,6 @@ public class RestApiTest { Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", @@ -229,6 +226,9 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), @@ -541,11 +541,13 @@ public class RestApiTest { " \"actualCpuCores\": {" + " \"createdMillis\": 1, " + " \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," + + " \"type\": \"HARD_FAIL\"," + " \"value\":2" + " }," + " \"diskSpace\": {" + " \"createdMillis\": 2, " + " \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," + + " \"type\": \"HARD_FAIL\"," + " \"details\": {" + " \"inGib\": 3," + " \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index ad5c28d6a80..af3552945d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -35,7 +35,7 @@ "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0", "failCount": 1, "wantToRetire": true, - "wantToDeprovision": true, + "wantToDeprovision": false, "history": [ { "event": "provisioned", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json index d33c1c9e743..a3d53798d7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -65,6 +65,7 @@ "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json index 4119e46e225..67b8d67c7f1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json @@ -30,7 +30,7 @@ "currentRebootGeneration": 0, "failCount": 0, "wantToRetire": false, - "wantToDeprovision": false, + "wantToDeprovision": true, "history": [ { "event": "provisioned", @@ -62,11 +62,13 @@ "actualCpuCores": { "createdMillis": 1, "description": "Actual number of CPU cores (2) differs from spec (4)", + "type":"HARD_FAIL", "value": 2 }, "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 3ebc87010fb..5dd4e7ea87d 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -42,6 +42,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>flags</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -59,6 +65,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index d1331b70251..10fa10f1150 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -50,12 +50,15 @@ public interface Orchestrator { HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException; /** - * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this. + * Returns a lambda, which when invoked with a hostname, returns its current host info. + * + * <p>When invoked multiple times, the hostname/host info mapping is not necessarily consistent. + * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, + * and when doing bulk reads.</p> * - * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads. * @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}. */ - Function<HostName, Optional<HostInfo>> getNodeStatuses(); + Function<HostName, Optional<HostInfo>> getHostResolver(); void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index d3bdaa6dc64..eb6a4119f8a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -1,13 +1,17 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; +import com.yahoo.log.LogLevel; import com.yahoo.time.TimeBudget; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.Optional; +import java.util.logging.Logger; /** * Context for an operation (or suboperation) of the Orchestrator that needs to pass through to the backend, @@ -15,29 +19,52 @@ import java.util.Optional; * * @author hakonhall */ -public class OrchestratorContext { +public class OrchestratorContext implements AutoCloseable { + private static final Logger logger = Logger.getLogger(OrchestratorContext.class.getName()); private static final Duration DEFAULT_TIMEOUT_FOR_SINGLE_OP = Duration.ofSeconds(10); private static final Duration DEFAULT_TIMEOUT_FOR_BATCH_OP = Duration.ofSeconds(60); private static final Duration TIMEOUT_OVERHEAD = Duration.ofMillis(500); + private final Optional<OrchestratorContext> parent; private final Clock clock; private final TimeBudget timeBudget; private final boolean probe; + private final boolean largeLocks; + private final boolean usePermanentlyDownStatus; + + // The key set is the set of applications locked by this context tree: Only the + // root context has a non-empty set. The value is an unlock callback to be called + // when the root context is closed. + private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>(); /** Create an OrchestratorContext for operations on multiple applications. */ - public static OrchestratorContext createContextForMultiAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), false); + public static OrchestratorContext createContextForMultiAppOp(Clock clock, boolean largeLocks) { + return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), + false, largeLocks, false); } /** Create an OrchestratorContext for an operation on a single application. */ public static OrchestratorContext createContextForSingleAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), false); + return createContextForSingleAppOp(clock, false); + } + + public static OrchestratorContext createContextForSingleAppOp(Clock clock, boolean usePermanentlyDownStatus) { + return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), + false, false, usePermanentlyDownStatus); } - private OrchestratorContext(Clock clock, TimeBudget timeBudget, boolean probe) { + private OrchestratorContext(OrchestratorContext parentOrNull, + Clock clock, + TimeBudget timeBudget, + boolean probe, + boolean largeLocks, + boolean usePermanentlyDownStatus) { + this.parent = Optional.ofNullable(parentOrNull); this.clock = clock; this.timeBudget = timeBudget; this.probe = probe; + this.largeLocks = largeLocks; + this.usePermanentlyDownStatus = usePermanentlyDownStatus; } public Duration getTimeLeft() { @@ -53,12 +80,50 @@ public class OrchestratorContext { return probe; } + /** Whether application locks acquired during probing of a batch suspend should be closed after the non-probe is done. */ + public boolean largeLocks() { return largeLocks; } + + /** Whether the PERMANENTLY_DOWN host status should be used (where appropriate). */ + public boolean usePermanentlyDownStatus() { return usePermanentlyDownStatus; } + + /** + * Returns true if 1. large locks is enabled, and 2. + * {@link #registerLockAcquisition(ApplicationInstanceReference, Runnable) registerLockAcquisition} + * has been invoked on any context below the root context that returned true. + */ + public boolean hasLock(ApplicationInstanceReference application) { + return parent.map(p -> p.hasLock(application)).orElseGet(() -> locks.containsKey(application)); + } + + /** + * Returns true if large locks is enabled in the root context, and in case the unlock callback + * will be invoked when the root context is closed. + */ + public boolean registerLockAcquisition(ApplicationInstanceReference application, Runnable unlock) { + if (parent.isPresent()) { + return parent.get().registerLockAcquisition(application, unlock); + } + + if (!largeLocks) { + return false; + } + + if (locks.containsKey(application)) { + unlock.run(); + throw new IllegalStateException("Application " + application + " was already associated with a lock"); + } + + locks.put(application, unlock); + + return true; + } + /** Create OrchestratorContext to use within an application lock. */ public OrchestratorContext createSubcontextWithinLock() { // Move deadline towards past by a fixed amount to ensure there's time to process exceptions and // access ZooKeeper before the lock times out. TimeBudget subTimeBudget = timeBudget.withDeadline(timeBudget.deadline().get().minus(TIMEOUT_OVERHEAD)); - return new OrchestratorContext(clock, subTimeBudget, probe); + return new OrchestratorContext(this, clock, subTimeBudget, probe, largeLocks, usePermanentlyDownStatus); } /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */ @@ -70,9 +135,18 @@ public class OrchestratorContext { deadline = maxDeadline; } - return new OrchestratorContext( - clock, - TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline))), - probe); + TimeBudget timeBudget = TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline))); + return new OrchestratorContext(this, clock, timeBudget, probe, largeLocks, usePermanentlyDownStatus); + } + + @Override + public void close() { + locks.forEach((application, unlock) -> { + try { + unlock.run(); + } catch (RuntimeException e) { + logger.log(LogLevel.ERROR, "Failed run on close : " + e.getMessage()); + } + }); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index f0b7c2eead1..414548f8bdc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -12,6 +12,10 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; @@ -28,6 +32,7 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.policy.Policy; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -59,13 +64,16 @@ public class OrchestratorImpl implements Orchestrator { private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; private final ApplicationApiFactory applicationApiFactory; + private final BooleanFlag enableLargeOrchestratorLocks; + private final BooleanFlag retireWithPermanentlyDownFlag; @Inject public OrchestratorImpl(ClusterControllerClientFactory clusterControllerClientFactory, StatusService statusService, OrchestratorConfig orchestratorConfig, InstanceLookupService instanceLookupService, - ConfigserverConfig configServerConfig) + ConfigserverConfig configServerConfig, + FlagSource flagSource) { this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, new ApplicationApiFactory(configServerConfig.zookeeperserver().size())), clusterControllerClientFactory, @@ -73,7 +81,8 @@ public class OrchestratorImpl implements Orchestrator { instanceLookupService, orchestratorConfig.serviceMonitorConvergenceLatencySeconds(), Clock.systemUTC(), - new ApplicationApiFactory(configServerConfig.zookeeperserver().size())); + new ApplicationApiFactory(configServerConfig.zookeeperserver().size()), + flagSource); } public OrchestratorImpl(Policy policy, @@ -82,7 +91,8 @@ public class OrchestratorImpl implements Orchestrator { InstanceLookupService instanceLookupService, int serviceMonitorConvergenceLatencySeconds, Clock clock, - ApplicationApiFactory applicationApiFactory) + ApplicationApiFactory applicationApiFactory, + FlagSource flagSource) { this.policy = policy; this.clusterControllerClientFactory = clusterControllerClientFactory; @@ -91,6 +101,8 @@ public class OrchestratorImpl implements Orchestrator { this.instanceLookupService = instanceLookupService; this.clock = clock; this.applicationApiFactory = applicationApiFactory; + this.enableLargeOrchestratorLocks = Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.bindTo(flagSource); + this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource); } @Override @@ -113,7 +125,7 @@ public class OrchestratorImpl implements Orchestrator { } @Override - public Function<HostName, Optional<HostInfo>> getNodeStatuses() { + public Function<HostName, Optional<HostInfo>> getHostResolver() { return hostName -> instanceLookupService.findInstanceByHost(hostName) .map(application -> statusService.getHostInfo(application.reference(), hostName)); } @@ -140,6 +152,12 @@ public class OrchestratorImpl implements Orchestrator { * monitoring will have had time to catch up. Since we don't want do the delay with the lock held, * and the host status service's locking functionality does not support something like condition * variables or Object.wait(), we break out here, releasing the lock before delaying. + * + * 2020-02-07: We should utilize suspendedSince timestamp on the HostInfo: The above + * is equivalent to guaranteeing a minimum time after suspendedSince, before checking + * the health with service monitor. This should for all practical purposes remove + * the amount of time in this sleep. + * Caveat: Cannot be implemented before lingering HostInfo has been fixed (VESPA-17546). */ sleep(serviceMonitorConvergenceLatencySeconds, TimeUnit.SECONDS); @@ -149,15 +167,21 @@ public class OrchestratorImpl implements Orchestrator { try (MutableStatusRegistry statusRegistry = statusService .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { HostStatus currentHostState = statusRegistry.getHostInfo(hostName).status(); - - if (HostStatus.NO_REMARKS == currentHostState) { + if (currentHostState == HostStatus.NO_REMARKS) { return; } - ApplicationInstanceStatus appStatus = statusRegistry.getStatus(); - if (appStatus == ApplicationInstanceStatus.NO_REMARKS) { - policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); + // In 2 cases the resume will appear to succeed (no exception thrown), + // but the host status and content cluster states will not be changed accordingly: + // 1. When host is permanently down: the host will be removed from the application asap. + // 2. The whole application is down: the content cluster states are set to maintenance, + // and the host may be taken down manually at any moment. + if (currentHostState == HostStatus.PERMANENTLY_DOWN || + statusRegistry.getStatus() == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { + return; } + + policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); } } @@ -173,7 +197,10 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstance appInstance = getApplicationInstance(hostName); NodeGroup nodeGroup = new NodeGroup(appInstance, hostName); - OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); + boolean usePermanentlyDownStatus = retireWithPermanentlyDownFlag + .with(FetchVector.Dimension.HOSTNAME, hostName.s()) + .value(); + OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock, usePermanentlyDownStatus); try (MutableStatusRegistry statusRegistry = statusService .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, statusRegistry, @@ -230,17 +257,20 @@ public class OrchestratorImpl implements Orchestrator { @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { - OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock); + boolean largeLocks = enableLargeOrchestratorLocks + .with(FetchVector.Dimension.HOSTNAME, parentHostname.s()) + .value(); + try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock, largeLocks)) { + List<NodeGroup> nodeGroupsOrderedByApplication; + try { + nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); + } catch (HostNameNotFoundException e) { + throw new BatchHostNameNotFoundException(parentHostname, hostNames, e); + } - List<NodeGroup> nodeGroupsOrderedByApplication; - try { - nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); - } catch (HostNameNotFoundException e) { - throw new BatchHostNameNotFoundException(parentHostname, hostNames, e); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false); } - - suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true); - suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false); } private void suspendAllNodeGroups(OrchestratorContext context, @@ -253,6 +283,8 @@ public class OrchestratorImpl implements Orchestrator { suspendGroup(context.createSubcontextForSingleAppOp(probe), nodeGroup); } catch (HostStateChangeDeniedException e) { throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e); + } catch (UncheckedTimeoutException e) { + throw e; } catch (RuntimeException e) { throw new BatchInternalErrorException(parentHostname, nodeGroup, e); } @@ -334,9 +366,15 @@ public class OrchestratorImpl implements Orchestrator { if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { ApplicationInstance application = getApplicationInstance(appRef); + HostInfos hostInfosSnapshot = statusRegistry.getHostInfos(); + // Mark it allowed to be down before we manipulate the clustercontroller OrchestratorUtil.getHostsUsedByApplicationInstance(application) - .forEach(h -> statusRegistry.setHostState(h, HostStatus.ALLOWED_TO_BE_DOWN)); + .stream() + // This filter also ensures host status is not modified if a suspended host + // has status != ALLOWED_TO_BE_DOWN. + .filter(hostname -> !hostInfosSnapshot.getOrNoRemarks(hostname).status().isSuspended()) + .forEach(hostname -> statusRegistry.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN)); // If the clustercontroller throws an error the nodes will be marked as allowed to be down // and be set back up on next resume invocation. diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java index 2e85713d323..a6353e39610 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.List; +import java.util.function.Predicate; /** * The API a Policy has access to @@ -29,10 +30,12 @@ public interface ApplicationApi { ApplicationInstanceStatus getApplicationStatus(); void setHostState(OrchestratorContext context, HostName hostName, HostStatus status); - List<HostName> getNodesInGroupWithStatus(HostStatus status); + List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate); + default List<HostName> getNodesInGroupWithStatus(HostStatus requiredStatus) { + return getNodesInGroupWith(status -> status == requiredStatus); + } List<StorageNode> getStorageNodesInGroupInClusterOrder(); List<StorageNode> getUpStorageNodesInGroupInClusterOrder(); - List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder(); - + List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index eb5dcf790ba..cf6946fa7f8 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; @@ -17,10 +18,9 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -34,7 +34,7 @@ public class ApplicationApiImpl implements ApplicationApi { private final NodeGroup nodeGroup; private final MutableStatusRegistry hostStatusService; private final List<ClusterApi> clusterInOrder; - private final Map<HostName, HostStatus> hostStatusMap; + private final HostInfos hostInfos; public ApplicationApiImpl(NodeGroup nodeGroup, MutableStatusRegistry hostStatusService, @@ -44,10 +44,8 @@ public class ApplicationApiImpl implements ApplicationApi { this.nodeGroup = nodeGroup; this.hostStatusService = hostStatusService; Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance); - this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(), - hostName -> hostStatusService.getSuspendedHosts().contains(hostName) - ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS)); - this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory, numberOfConfigServers); + this.hostInfos = hostStatusService.getHostInfos(); + this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers); } @Override @@ -56,7 +54,7 @@ public class ApplicationApiImpl implements ApplicationApi { } private HostStatus getHostStatus(HostName hostName) { - return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS); + return hostInfos.getOrNoRemarks(hostName).status(); } @Override @@ -65,9 +63,9 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override - public List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() { + public List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder() { return getStorageNodesInGroupInClusterOrder().stream() - .filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) + .filter(storageNode -> getHostStatus(storageNode.hostName()).isSuspended()) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); } @@ -106,13 +104,14 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override - public List<HostName> getNodesInGroupWithStatus(HostStatus status) { + public List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate) { return nodeGroup.getHostNames().stream() - .filter(hostName -> getHostStatus(hostName) == status) + .filter(hostName -> statusPredicate.test(getHostStatus(hostName))) .collect(Collectors.toList()); } - private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, Map<HostName, HostStatus> hostStatusMap, + private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, + HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, int numberOfConfigServers) { Set<ServiceCluster> clustersInGroup = getServiceClustersInGroup(nodeGroup); @@ -121,7 +120,7 @@ public class ApplicationApiImpl implements ApplicationApi { this, serviceCluster, nodeGroup, - hostStatusMap, + hostInfos, clusterControllerClientFactory, numberOfConfigServers)) .sorted(ApplicationApiImpl::compareClusters) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 2aaebb18aa6..b747d8c2e22 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.Collections; @@ -27,7 +28,7 @@ class ClusterApiImpl implements ClusterApi { private final ApplicationApi applicationApi; private final ServiceCluster serviceCluster; private final NodeGroup nodeGroup; - private final Map<HostName, HostStatus> hostStatusMap; + private final HostInfos hostInfos; private final ClusterControllerClientFactory clusterControllerClientFactory; private final Set<ServiceInstance> servicesInGroup; private final Set<ServiceInstance> servicesDownInGroup; @@ -50,13 +51,13 @@ class ClusterApiImpl implements ClusterApi { public ClusterApiImpl(ApplicationApi applicationApi, ServiceCluster serviceCluster, NodeGroup nodeGroup, - Map<HostName, HostStatus> hostStatusMap, + HostInfos hostInfos, ClusterControllerClientFactory clusterControllerClientFactory, int numberOfConfigServers) { this.applicationApi = applicationApi; this.serviceCluster = serviceCluster; this.nodeGroup = nodeGroup; - this.hostStatusMap = hostStatusMap; + this.hostInfos = hostInfos; this.clusterControllerClientFactory = clusterControllerClientFactory; Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality = @@ -144,7 +145,7 @@ class ClusterApiImpl implements ClusterApi { public String nodesAllowedToBeDownNotInGroupDescription() { return servicesNotInGroup.stream() .map(ServiceInstance::hostName) - .filter(hostName -> hostStatus(hostName) == HostStatus.ALLOWED_TO_BE_DOWN) + .filter(hostName -> hostStatus(hostName).isSuspended()) .sorted() .distinct() .collect(Collectors.toList()) @@ -206,11 +207,11 @@ class ClusterApiImpl implements ClusterApi { } private HostStatus hostStatus(HostName hostName) { - return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS); + return hostInfos.getOrNoRemarks(hostName).status(); } private boolean serviceEffectivelyDown(ServiceInstance service) { - if (hostStatus(service.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) { + if (hostStatus(service.hostName()).isSuspended()) { return true; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index a9b9736ebfb..f6a1e4f91f0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -30,7 +30,9 @@ public class HostedVespaPolicy implements Policy { private final ClusterControllerClientFactory clusterControllerClientFactory; private final ApplicationApiFactory applicationApiFactory; - public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory, ApplicationApiFactory applicationApiFactory) { + public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, + ClusterControllerClientFactory clusterControllerClientFactory, + ApplicationApiFactory applicationApiFactory) { this.clusterPolicy = clusterPolicy; this.clusterControllerClientFactory = clusterControllerClientFactory; this.applicationApiFactory = applicationApiFactory; @@ -60,10 +62,11 @@ public class HostedVespaPolicy implements Policy { public void releaseSuspensionGrant(OrchestratorContext context, ApplicationApi application) throws HostStateChangeDeniedException { // Always defer to Cluster Controller whether it's OK to resume storage node - for (StorageNode storageNode : application.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder()) { + for (StorageNode storageNode : application.getSuspendedStorageNodesInGroupInReverseClusterOrder()) { storageNode.setNodeState(context, ClusterControllerNodeState.UP); } + // In particular, we're not modifying the state of PERMANENTLY_DOWN nodes. for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.ALLOWED_TO_BE_DOWN)) { application.setHostState(context, hostName, HostStatus.NO_REMARKS); } @@ -92,9 +95,15 @@ public class HostedVespaPolicy implements Policy { storageNode.setNodeState(context, ClusterControllerNodeState.DOWN); } - // Ensure all nodes in the group are marked as allowed to be down - for (HostName hostName : applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { - applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); + if (context.usePermanentlyDownStatus()) { + // Ensure all nodes in the group are marked as permanently down + for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.PERMANENTLY_DOWN)) { + applicationApi.setHostState(context, hostName, HostStatus.PERMANENTLY_DOWN); + } + } else { + for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.ALLOWED_TO_BE_DOWN)) { + applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); + } } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index bd265ed39e4..7b8a74d7fe2 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.service.manager.MonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; @@ -81,12 +81,11 @@ public class InstanceResource { = instanceLookupService.findInstanceById(instanceId) .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build())); - Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference()); + HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference()); Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance) .stream() .collect(Collectors.toMap(hostName -> hostName, - hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name() - : HostStatus.NO_REMARKS.name())); + hostName -> hostInfos.getOrNoRemarks(hostName).status().asString())); return InstanceStatusResponse.create(applicationInstance, hostStatusMap); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java index c2ea0c9eddf..068423f7d24 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.util.Map; import java.util.Objects; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java index fe78a41672b..1acd82662a4 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java @@ -8,10 +8,11 @@ import java.util.Set; import java.util.stream.Collectors; /** - * Collection of suspended hosts. + * Collection of the suspended hosts of an application. * * @author hakonhall */ +// @Immutable public class HostInfos { private final Map<HostName, HostInfo> hostInfos; @@ -23,16 +24,8 @@ public class HostInfos { this.hostInfos = Map.of(); } - /** Get all suspended hostnames. */ - public Set<HostName> suspendedHostsnames() { - return hostInfos.entrySet().stream() - .filter(entry -> entry.getValue().status().isSuspended()) - .map(entry -> entry.getKey()) - .collect(Collectors.toSet()); - } - /** Get host info for hostname, returning a NO_REMARKS HostInfo if unknown. */ - public HostInfo get(HostName hostname) { + public HostInfo getOrNoRemarks(HostName hostname) { return hostInfos.getOrDefault(hostname, HostInfo.createNoRemarks()); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java index 9998e7de918..680f3cbcc6d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java @@ -28,17 +28,24 @@ public class HostInfosCache implements HostInfosService { this.cacheGeneration = new AtomicLong(counter.get()); } - @Override - public HostInfos getHostInfos(ApplicationInstanceReference application) { + public void refreshCache() { long newCacheGeneration = counter.get(); if (cacheGeneration.getAndSet(newCacheGeneration) != newCacheGeneration) { suspendedHosts.clear(); } + } + public HostInfos getCachedHostInfos(ApplicationInstanceReference application) { return suspendedHosts.computeIfAbsent(application, wrappedService::getHostInfos); } @Override + public HostInfos getHostInfos(ApplicationInstanceReference application) { + refreshCache(); + return getCachedHostInfos(application); + } + + @Override public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) { boolean isException = true; boolean modified = false; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java index 6764ffb48ea..330e53f9586 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java @@ -23,4 +23,5 @@ public enum HostStatus { HostStatus(boolean suspended) { this.suspended = suspended; } public boolean isSuspended() { return suspended; } + public String asString() { return name(); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java index 92b0ec50011..d2042dc9fd2 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java @@ -14,22 +14,16 @@ import java.util.Set; */ public interface MutableStatusRegistry extends AutoCloseable { - /** - * Returns the status of this application. - */ + /** Returns the status of this application. */ ApplicationInstanceStatus getStatus(); /** Returns the host info of the given host. */ HostInfo getHostInfo(HostName hostName); - /** - * Returns the set of all suspended hosts for this application. - */ - Set<HostName> getSuspendedHosts(); + /** Returns a snapshot of all host infos for this application. */ + HostInfos getHostInfos(); - /** - * Sets the state for the given host. - */ + /** Sets the state for the given host. */ void setHostState(HostName hostName, HostStatus status); /** diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index 9e3ee84e1d9..e2be5ec7eb6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -57,12 +57,12 @@ public interface StatusService { Set<ApplicationInstanceReference> getAllSuspendedApplications(); /** - * Returns a fresh, but not necessarily consistent mapping from applications to their set of suspended hosts. + * Returns a lambda, which when invoked for an application, returns an up-to-date snapshot of {@link HostInfos host infos}. * - * If the lock for an application is held when this mapping is acquired, new sets returned for that application - * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.) + * <p>Unless the lock for the application is held, the returned snapshot may already be out of date. + * (The snapshot itself is immutable.)</p> */ - Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication(); + Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver(); /** Returns the status of the given application. This is consistent if its lock is held.*/ ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index c4a23baae2e..b4025167187 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -70,11 +70,19 @@ public class ZookeeperStatusService implements StatusService { @Override public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) { - return ZookeeperStatusService.this.setHostStatusInZk(application, hostName, hostStatus); + return ZookeeperStatusService.this.setHostInfoInZk(application, hostName, hostStatus); } }); } + /** Non-private for testing only. */ + ZookeeperStatusService(Curator curator, Metric metric, Timer timer, HostInfosCache hostInfosCache) { + this.curator = curator; + this.metric = metric; + this.timer = timer; + this.hostInfosCache = hostInfosCache; + } + @Override public Set<ApplicationInstanceReference> getAllSuspendedApplications() { try { @@ -99,12 +107,13 @@ public class ZookeeperStatusService implements StatusService { /** * Cache is checked for freshness when this mapping is created, and may be invalidated again later - * by other users of the cache. Since this function is backed by the cache, any such invalidations + * by other users of the cache. Since this function is backed by the cache, any such invalidation * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it. */ @Override - public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() { - return application -> hostInfosCache.getHostInfos(application).suspendedHostsnames(); + public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() { + hostInfosCache.refreshCache(); + return hostInfosCache::getCachedHostInfos; } @@ -122,6 +131,35 @@ public class ZookeeperStatusService implements StatusService { public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( OrchestratorContext context, ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException { + Runnable onRegistryClose; + + // A multi-application operation, aka batch suspension, will first issue a probe + // then a non-probe. With "large locks", the lock is not release in between - + // no lock is taken on the non-probe. Instead, the release is done on the multi-application + // context close. + if (context.hasLock(applicationInstanceReference)) { + onRegistryClose = () -> {}; + } else { + Runnable unlock = acquireLock(context, applicationInstanceReference); + if (context.registerLockAcquisition(applicationInstanceReference, unlock)) { + onRegistryClose = () -> {}; + } else { + onRegistryClose = unlock; + } + } + + try { + return new ZkMutableStatusRegistry(onRegistryClose, applicationInstanceReference, context.isProbe()); + } catch (Throwable t) { + // In case the constructor throws an exception. + onRegistryClose.run(); + throw t; + } + } + + private Runnable acquireLock(OrchestratorContext context, + ApplicationInstanceReference applicationInstanceReference) + throws UncheckedTimeoutException { ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference); String app = applicationId.application().value() + "." + applicationId.instance().value(); Map<String, String> dimensions = Map.of( @@ -151,89 +189,66 @@ public class ZookeeperStatusService implements StatusService { metric.add(acquireResultMetricName, 1, metricContext); } - Runnable updateLockHoldMetric = () -> { + return () -> { + try { + lock.close(); + } catch (RuntimeException e) { + // We may want to avoid logging some exceptions that may be expected, like when session expires. + log.log(LogLevel.WARNING, + "Failed to close application lock for " + + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", + e); + } + Instant lockReleasedTime = timer.currentTime(); double seconds = durationInSeconds(acquireEndTime, lockReleasedTime); metric.set("orchestrator.lock.hold-latency", seconds, metricContext); }; - - try { - return new ZkMutableStatusRegistry(lock, applicationInstanceReference, context.isProbe(), updateLockHoldMetric); - } catch (Throwable t) { - // In case the constructor throws an exception. - updateLockHoldMetric.run(); - lock.close(); - throw t; - } } private double durationInSeconds(Instant startInstant, Instant endInstant) { return Duration.between(startInstant, endInstant).toMillis() / 1000.0; } - /** Do not call this directly: should be called behind a cache. */ - private boolean setHostStatusInZk(ApplicationInstanceReference applicationInstanceReference, - HostName hostName, - HostStatus status) { - String hostAllowedDownPath = hostAllowedDownPath(applicationInstanceReference, hostName); - - boolean modified = false; - try { - switch (status) { - case NO_REMARKS: - // Deprecated: Remove once 7.170 has rolled out to infrastructure - modified = deleteNode_ignoreNoNodeException(hostAllowedDownPath, "Host already has state NO_REMARKS, path = " + hostAllowedDownPath); - break; - default: - // ignore, e.g. ALLOWED_TO_BE_DOWN should NOT create a new deprecated znode. - } - - modified |= setHostInfoInZk(applicationInstanceReference, hostName, status); - } catch (Exception e) { - throw new RuntimeException(e); - } - - return modified; - } - /** Returns false if no changes were made. */ - private boolean setHostInfoInZk(ApplicationInstanceReference application, HostName hostname, HostStatus status) - throws Exception { + private boolean setHostInfoInZk(ApplicationInstanceReference application, HostName hostname, HostStatus status) { String path = hostPath(application, hostname); if (status == HostStatus.NO_REMARKS) { return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path); } - Optional<HostInfo> currentHostInfo = readBytesFromZk(path).map(WireHostInfo::deserialize); + Optional<HostInfo> currentHostInfo = uncheck(() -> readBytesFromZk(path)).map(WireHostInfo::deserialize); if (currentHostInfo.isEmpty()) { Instant suspendedSince = timer.currentTime(); HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); - curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes); + uncheck(() -> curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes)); } else if (currentHostInfo.get().status() == status) { return false; } else { Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime); HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); - curator.framework().setData().forPath(path, hostInfoBytes); + uncheck(() -> curator.framework().setData().forPath(path, hostInfoBytes)); } return true; } - private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception { + private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) { try { curator.framework().delete().forPath(path); return true; } catch (NoNodeException e) { log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); return false; + } catch (Exception e) { + throw new RuntimeException(e); } } - private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) throws Exception { + private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) { try { curator.framework().create() .creatingParentsIfNeeded() @@ -242,6 +257,8 @@ public class ZookeeperStatusService implements StatusService { } catch (NodeExistsException e) { log.log(LogLevel.DEBUG, debugLogMessageIfExists, e); return false; + } catch (Exception e) { + throw new RuntimeException(e); } } @@ -255,7 +272,7 @@ public class ZookeeperStatusService implements StatusService { @Override public HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName) { - return hostInfosCache.getHostInfos(applicationInstanceReference).get(hostName); + return hostInfosCache.getHostInfos(applicationInstanceReference).getOrNoRemarks(hostName); } /** Do not call this directly: should be called behind a cache. */ @@ -336,19 +353,16 @@ public class ZookeeperStatusService implements StatusService { private class ZkMutableStatusRegistry implements MutableStatusRegistry { - private final Lock lock; + private final Runnable onClose; private final ApplicationInstanceReference applicationInstanceReference; private final boolean probe; - private final Runnable onLockRelease; - public ZkMutableStatusRegistry(Lock lock, + public ZkMutableStatusRegistry(Runnable onClose, ApplicationInstanceReference applicationInstanceReference, - boolean probe, - Runnable onLockRelease) { - this.lock = lock; + boolean probe) { + this.onClose = onClose; this.applicationInstanceReference = applicationInstanceReference; this.probe = probe; - this.onLockRelease = onLockRelease; } @Override @@ -362,8 +376,8 @@ public class ZookeeperStatusService implements StatusService { } @Override - public Set<HostName> getSuspendedHosts() { - return hostInfosCache.getHostInfos(applicationInstanceReference).suspendedHostsnames(); + public HostInfos getHostInfos() { + return hostInfosCache.getHostInfos(applicationInstanceReference); } @Override @@ -380,31 +394,26 @@ public class ZookeeperStatusService implements StatusService { log.log(LogLevel.INFO, "Setting app " + applicationInstanceReference.asString() + " to status " + applicationInstanceStatus); String path = applicationInstanceSuspendedPath(applicationInstanceReference); - try { - switch (applicationInstanceStatus) { - case NO_REMARKS: - deleteNode_ignoreNoNodeException(path, - "Instance is already in state NO_REMARKS, path = " + path); - break; - case ALLOWED_TO_BE_DOWN: - createNode_ignoreNodeExistsException(path, - "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path); - break; - } - } catch (Exception e) { - throw new RuntimeException(e); + switch (applicationInstanceStatus) { + case NO_REMARKS: + deleteNode_ignoreNoNodeException(path, + "Instance is already in state NO_REMARKS, path = " + path); + break; + case ALLOWED_TO_BE_DOWN: + createNode_ignoreNodeExistsException(path, + "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path); + break; } } @Override public void close() { - onLockRelease.run(); try { - lock.close(); + onClose.run(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. log.log(LogLevel.WARNING, - "Failed to close application lock for " + + "Failed close application lock in " + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", e); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java new file mode 100644 index 00000000000..607894ee104 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java @@ -0,0 +1,59 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator; + +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.TenantId; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class OrchestratorContextTest { + private final ApplicationInstanceReference application = new ApplicationInstanceReference( + new TenantId("tenant"), + new ApplicationInstanceId("app:dev:us-east-1:default")); + + @Test + public void testLargeLocks() { + var mutable = new Object() { boolean locked = true; }; + Runnable unlock = () -> mutable.locked = false; + + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), true)) { + try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { + assertFalse(probeContext.hasLock(application)); + assertTrue(probeContext.registerLockAcquisition(application, unlock)); + + assertTrue(probeContext.hasLock(application)); + assertTrue(mutable.locked); + } + + try (OrchestratorContext nonProbeContext = rootContext.createSubcontextForSingleAppOp(false)) { + assertTrue(nonProbeContext.hasLock(application)); + assertTrue(mutable.locked); + } + + assertTrue(mutable.locked); + } + assertFalse(mutable.locked); + } + + @Test + public void testLargeLocksDisabled() { + var mutable = new Object() { boolean locked = true; }; + Runnable unlock = () -> mutable.locked = false; + + try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), false)) { + try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) { + assertFalse(probeContext.hasLock(application)); + assertFalse(probeContext.registerLockAcquisition(application, unlock)); + } + } + + assertTrue(mutable.locked); + } +}
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 45d4c531898..a2c99b86ae2 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -17,6 +17,8 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; @@ -26,6 +28,7 @@ import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; import com.yahoo.vespa.service.monitor.ServiceModel; @@ -36,7 +39,9 @@ import org.mockito.InOrder; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; @@ -55,6 +60,11 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce; /** * Test Orchestrator with a mock backend (the MockCurator) @@ -64,6 +74,7 @@ import static org.mockito.Mockito.spy; public class OrchestratorImplTest { private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private ApplicationId app1; private ApplicationId app2; @@ -88,7 +99,8 @@ public class OrchestratorImplTest { new DummyInstanceLookupService(), 0, new ManualClock(), - applicationApiFactory); + applicationApiFactory, + flagSource); clustercontroller.setAllDummyNodesAsUp(); } @@ -312,6 +324,72 @@ public class OrchestratorImplTest { } @Test + public void testLargeLocks() throws Exception { + flagSource.withBooleanFlag(Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.id(), true); + + var tenantId = new TenantId("tenant"); + var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default"); + var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId); + + var policy = mock(HostedVespaPolicy.class); + var zookeeperStatusService = mock(ZookeeperStatusService.class); + var instanceLookupService = mock(InstanceLookupService.class); + var applicationInstance = mock(ApplicationInstance.class); + var clusterControllerClientFactory = mock(ClusterControllerClientFactory.class); + var clock = new ManualClock(); + var applicationApiFactory = mock(ApplicationApiFactory.class); + var hostStatusRegistry = mock(MutableStatusRegistry.class); + + when(instanceLookupService.findInstanceByHost(any())).thenReturn(Optional.of(applicationInstance)); + when(applicationInstance.reference()).thenReturn(applicationInstanceReference); + when(zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(any(), any())) + .thenReturn(hostStatusRegistry); + when(hostStatusRegistry.getStatus()).thenReturn(NO_REMARKS); + + var orchestrator = new OrchestratorImpl( + policy, + clusterControllerClientFactory, + zookeeperStatusService, + instanceLookupService, + 20, + clock, + applicationApiFactory, + flagSource); + + HostName parentHostname = new HostName("parent.vespa.ai"); + + orchestrator.suspendAll(parentHostname, List.of(parentHostname)); + + ArgumentCaptor<OrchestratorContext> contextCaptor = ArgumentCaptor.forClass(OrchestratorContext.class); + verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(contextCaptor.capture(), any()); + List<OrchestratorContext> contexts = contextCaptor.getAllValues(); + + // First invocation is probe, second is not. + assertEquals(2, contexts.size()); + assertTrue(contexts.get(0).isProbe()); + assertTrue(contexts.get(0).largeLocks()); + assertFalse(contexts.get(1).isProbe()); + assertTrue(contexts.get(1).largeLocks()); + + verify(applicationApiFactory, times(2)).create(any(), any(), any()); + verify(policy, times(2)).grantSuspensionRequest(any(), any()); + verify(instanceLookupService, atLeastOnce()).findInstanceByHost(any()); + verify(hostStatusRegistry, times(2)).getStatus(); + + // Each zookeeperStatusService that is created, is closed. + verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(any(), any()); + verify(hostStatusRegistry, times(2)).close(); + + verifyNoMoreInteractions( + policy, + clusterControllerClientFactory, + zookeeperStatusService, + hostStatusRegistry, + instanceLookupService, + applicationApiFactory); + } + + @Test public void testGetHost() throws Exception { ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); @@ -349,7 +427,8 @@ public class OrchestratorImplTest { lookupService, 0, new ManualClock(), - applicationApiFactory); + applicationApiFactory, + flagSource); orchestrator.setNodeStatus(hostName, HostStatus.ALLOWED_TO_BE_DOWN); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java index d5734a73de0..ee33a92367b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java @@ -327,7 +327,7 @@ public class ApplicationApiImplTest { private void verifyStorageNodesAllowedToBeDown( ApplicationApi applicationApi, HostName... hostNames) { List<HostName> actualStorageNodes = - applicationApi.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder().stream() + applicationApi.getSuspendedStorageNodesInGroupInReverseClusterOrder().stream() .map(storageNode -> storageNode.hostName()) .collect(Collectors.toList()); assertEquals(Arrays.asList(hostNames), actualStorageNodes); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 8e42ccdc79d..62925dc003e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import org.junit.Test; @@ -75,7 +76,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo()); @@ -184,7 +185,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp()); @@ -214,7 +215,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(applicationInstance, hostName1, hostName3), - new HashMap<>(), + new HostInfos(), modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS); assertTrue(clusterApi.isStorageCluster()); @@ -254,7 +255,7 @@ public class ClusterApiImplTest { applicationApi, serviceCluster, new NodeGroup(application, hostnames.get(0)), - modelUtils.getHostStatusMap(), + modelUtils.getHostInfos(), modelUtils.getClusterControllerClientFactory(), clusterSize); assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices()); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index 729b3ae79ff..eff222bc074 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.OrchestratorContext; @@ -25,6 +26,8 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -33,11 +36,13 @@ import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.yolean.Exceptions; import java.time.Clock; +import java.time.Instant; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.mockito.Mockito.mock; @@ -48,6 +53,7 @@ class ModelTestUtils { public static final int NUMBER_OF_CONFIG_SERVERS = 3; + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>(); private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>(); @@ -58,14 +64,30 @@ class ModelTestUtils { new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)), 0, new ManualClock(), - applicationApiFactory()); + applicationApiFactory(), + flagSource); ApplicationApiFactory applicationApiFactory() { return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS); } - Map<HostName, HostStatus> getHostStatusMap() { - return hostStatusMap; + HostInfos getHostInfos() { + Instant now = Instant.now(); + + Map<HostName, HostInfo> hostInfosMap = hostStatusMap.entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> { + HostStatus status = entry.getValue(); + if (status == HostStatus.NO_REMARKS) { + return HostInfo.createNoRemarks(); + } else { + return HostInfo.createSuspended(status, now); + } + } + )); + + return new HostInfos(hostInfosMap); } HostName createNode(String name, HostStatus hostStatus) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index b27e37ac034..ed6917a3a4e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -120,7 +120,7 @@ public class HostedVespaPolicyTest { when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); - when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames); + when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames); InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); @@ -136,7 +136,7 @@ public class HostedVespaPolicyTest { order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN); order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN); - order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS); + order.verify(applicationApi).getNodesInGroupWith(any()); order.verify(applicationApi).setHostState(context, hostName1, HostStatus.ALLOWED_TO_BE_DOWN); order.verify(applicationApi).setHostState(context, hostName2, HostStatus.ALLOWED_TO_BE_DOWN); order.verify(applicationApi).setHostState(context, hostName3, HostStatus.ALLOWED_TO_BE_DOWN); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index 89f421e9125..80d0af09792 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -155,9 +155,11 @@ public class ApplicationSuspensionResourceTest { return "<services>\n" + " <container version=\"1.0\" jetty=\"true\">\n" + + " <accesslog type=\"disabled\"/>\n" + " <config name=\"container.handler.threadpool\">\n" + " <maxthreads>10</maxthreads>\n" + " </config>\n" + + " <component id=\"com.yahoo.vespa.flags.InMemoryFlagSource\" bundle=\"flags\" />\n" + " <component id=\"com.yahoo.vespa.curator.mock.MockCurator\" bundle=\"zkfacade\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.status.ZookeeperStatusService\" bundle=\"orchestrator\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" + diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index fec1554396d..b19f96a5867 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.Host; @@ -90,6 +91,8 @@ public class HostResourceTest { makeServiceClusterSet()))); } + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() { @Override public Optional<ApplicationInstance> findInstanceById( @@ -129,23 +132,23 @@ public class HostResourceTest { } } - private static final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( + private final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory - ); + applicationApiFactory, + flagSource); - private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( + private final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory - ); + applicationApiFactory, + flagSource); private final UriInfo uriInfo = mock(UriInfo.class); @@ -247,7 +250,8 @@ public class HostResourceTest { EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory); + applicationApiFactory, + flagSource); try { HostResource hostResource = new HostResource(alwaysRejectResolver, uriInfo); @@ -267,7 +271,8 @@ public class HostResourceTest { mockInstanceLookupService, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, - applicationApiFactory); + applicationApiFactory, + flagSource); try { HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java new file mode 100644 index 00000000000..8f530f4abf3 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java @@ -0,0 +1,127 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Timer; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.orchestrator.OrchestratorContext; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class ZookeeperStatusService2Test { + private final Curator curator = mock(Curator.class); + private final Timer timer = new TestTimer(); + private final Metric metric = mock(Metric.class); + private final HostInfosCache cache = mock(HostInfosCache.class); + private final ZookeeperStatusService zookeeperStatusService = new ZookeeperStatusService(curator, metric, timer, cache); + + private final OrchestratorContext context = mock(OrchestratorContext.class); + private final InterProcessMutex mutex = mock(InterProcessMutex.class); + private final ApplicationInstanceReference reference = new ApplicationInstanceReference( + new TenantId("tenant"), new ApplicationInstanceId("app:dev:us-east-1:default")); + + @Test + public void verifyLocks() throws Exception { + when(context.isProbe()).thenReturn(true); + when(context.hasLock(any())).thenReturn(false); + when(context.registerLockAcquisition(any(), any())).thenReturn(false); + + when(curator.createMutex(any())).thenReturn(mutex); + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(curator, times(1)).createMutex(any()); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(1)).release(); + verify(context, times(1)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Now the non-probe suspension + + when(context.isProbe()).thenReturn(false); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(mutex, times(2)).acquire(anyLong(), any()); + verify(mutex, times(2)).release(); + ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class); + verify(context, times(2)).hasLock(any()); + verify(context, times(2)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + } + + @Test + public void verifyLargeLocks() throws Exception { + when(context.isProbe()).thenReturn(true); + when(context.hasLock(any())).thenReturn(false); + when(context.registerLockAcquisition(any(), any())).thenReturn(true); + + when(curator.createMutex(any())).thenReturn(mutex); + when(mutex.acquire(anyLong(), any())).thenReturn(true); + + when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + verify(curator, times(1)).createMutex(any()); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(0)).release(); + verify(context, times(1)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Now the non-probe suspension + + when(context.isProbe()).thenReturn(false); + when(context.hasLock(any())).thenReturn(true); + when(context.registerLockAcquisition(any(), any())).thenReturn(false); + + try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + // nothing + } + + // No (additional) acquire, and no releases. + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(0)).release(); + verify(context, times(2)).hasLock(any()); + verify(context, times(1)).registerLockAcquisition(any(), any()); + verifyNoMoreInteractions(mutex); + + // Verify the context runnable releases the mutex + ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class); + verify(context, times(1)).registerLockAcquisition(any(), runnableCaptor.capture()); + assertEquals(1, runnableCaptor.getAllValues().size()); + runnableCaptor.getAllValues().forEach(Runnable::run); + verify(mutex, times(1)).acquire(anyLong(), any()); + verify(mutex, times(1)).release(); + } +}
\ No newline at end of file diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h index 90484f8cc3a..c802730e6fe 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h @@ -17,19 +17,19 @@ class IConstAttributeFunctor { public: virtual void operator()(const IAttributeVector &attributeVector) = 0; - virtual ~IConstAttributeFunctor() { } + virtual ~IConstAttributeFunctor() = default; }; class IAttributeFunctor { public: virtual void operator()(IAttributeVector &attributeVector) = 0; - virtual ~IAttributeFunctor() { } + virtual ~IAttributeFunctor() = default; }; class IAttributeExecutor { public: - virtual ~IAttributeExecutor() { } + virtual ~IAttributeExecutor() = default; virtual void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const = 0; }; diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index c6d49c479c4..44ff8cb925e 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -215,7 +215,7 @@ struct SequentialAttributeManager { mgr.addInitializedAttributes(initializer.getInitializedAttributes()); } - ~SequentialAttributeManager() {} + ~SequentialAttributeManager() = default; }; struct DummyInitializerTask : public InitializerTask @@ -262,23 +262,33 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria initializer::TaskRunner taskRunner(executor); taskRunner.runTask(initializer); } -ParallelAttributeManager::~ParallelAttributeManager() {} +ParallelAttributeManager::~ParallelAttributeManager() = default; TEST_F("require that attributes are added", Fixture) { - EXPECT_TRUE(f.addAttribute("a1").get() != NULL); - EXPECT_TRUE(f.addAttribute("a2").get() != NULL); + EXPECT_TRUE(f.addAttribute("a1").get() != nullptr); + EXPECT_TRUE(f.addAttribute("a2").get() != nullptr); EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName()); EXPECT_EQUAL("a1", (*f._m.getAttributeReadGuard("a1", true))->getName()); EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName()); EXPECT_EQUAL("a2", (*f._m.getAttributeReadGuard("a2", true))->getName()); EXPECT_TRUE(!f._m.getAttribute("not")->valid()); + + auto rv = f._m.readable_attribute_vector("a1"); + ASSERT_TRUE(rv.get() != nullptr); + EXPECT_EQUAL("a1", rv->makeReadGuard(true)->attribute()->getName()); + + rv = f._m.readable_attribute_vector("a2"); + ASSERT_TRUE(rv.get() != nullptr); + EXPECT_EQUAL("a2", rv->makeReadGuard(true)->attribute()->getName()); + + EXPECT_TRUE(f._m.readable_attribute_vector("not_valid").get() == nullptr); } TEST_F("require that predicate attributes are added", Fixture) { EXPECT_TRUE(f._m.addAttribute({"p1", AttributeUtils::getPredicateConfig()}, - createSerialNum).get() != NULL); + createSerialNum).get() != nullptr); EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName()); EXPECT_EQUAL("p1", (*f._m.getAttributeReadGuard("p1", true))->getName()); } @@ -376,7 +386,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); EXPECT_EQUAL(1u, a1->getNumDocs()); - PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = static_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -396,7 +406,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); // loaded EXPECT_EQUAL(2u, a1->getNumDocs()); - PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + auto &pa = static_cast<PredicateAttribute &>(*a1); PredicateIndex &index = pa.getIndex(); uint32_t doc_id; a1->addDoc(doc_id); @@ -746,7 +756,7 @@ TEST_F("require that we can acquire exclusive read access to attribute", Fixture EXPECT_TRUE(noneAccessor.get() == nullptr); } -TEST_F("require that imported attributes are exposed via attribute context together vi regular attributes", Fixture) +TEST_F("require that imported attributes are exposed via attribute context together with regular attributes", Fixture) { f.addAttribute("attr"); f.addImportedAttribute("imported"); @@ -767,6 +777,17 @@ TEST_F("require that imported attributes are exposed via attribute context toget EXPECT_EQUAL("imported", all[1]->getName()); } +TEST_F("imported attributes are transparently returned from readable_attribute_vector", Fixture) +{ + f.addAttribute("attr"); + f.addImportedAttribute("imported"); + f.setImportedAttributes(); + auto av = f._m.readable_attribute_vector("imported"); + ASSERT_TRUE(av); + auto g = av->makeReadGuard(false); + EXPECT_EQUAL("imported", g->attribute()->getName()); +} + TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) { AVConfig generic_tensor(BasicType::TENSOR); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 14b72c9d8f8..edb5a07b059 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -20,6 +20,7 @@ #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcorespi/flush/iflushtarget.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> @@ -588,8 +589,8 @@ struct FilterFixture TEST_F("require that filter attribute manager can filter attributes", FilterFixture) { - EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL); - EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL); + EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr); + EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr); std::vector<AttributeGuard> attrs; f._filterMgr.getAttributeList(attrs); EXPECT_EQUAL(1u, attrs.size()); @@ -607,6 +608,16 @@ TEST_F("require that filter attribute manager can return flushed serial number", EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); } +TEST_F("readable_attribute_vector filters attributes", FilterFixture) +{ + auto av = f._filterMgr.readable_attribute_vector("a2"); + ASSERT_TRUE(av); + EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName()); + + av = f._filterMgr.readable_attribute_vector("a1"); + EXPECT_FALSE(av); +} + namespace { Tensor::UP make_tensor(const TensorSpec &spec) { diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index 5b1fa3ed4bf..744159962ad 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -75,7 +75,9 @@ makeDocTypeRepo() addField("aaa", Array(DataType::T_INT)). addField("aaw", Wset(DataType::T_INT)). addField("ab", DataType::T_INT). - addField("ae", DataType::T_INT)); + addField("ae", DataType::T_INT)). + imported_field("my_imported_field"). + imported_field("my_missing_imported_field"); builder.document(doc_type_id + 1, type_name_2, Struct(header_name_2), Struct(body_name_2). addField("ic", DataType::T_STRING). @@ -150,6 +152,9 @@ TestFixture::TestFixture() _amgr.addAttribute("aaa", AttributeFactory::createAttribute("aaa", { BasicType::INT32 , CollectionType::ARRAY})); _amgr.addAttribute("aaw", AttributeFactory::createAttribute("aaw", { BasicType::INT32 , CollectionType::WSET})); _amgr.addAttribute("ae", AttributeFactory::createAttribute("ae", { BasicType::INT32 })); + // We "fake" having an imported attribute to avoid having to set up reference attributes, mappings etc. + // This is fine since the attribute manager already abstracts away if an attribute is imported or not. + _amgr.addAttribute("my_imported_field", AttributeFactory::createAttribute("my_imported_field", { BasicType::INT32 })); _repoUP = makeDocTypeRepo(); } @@ -170,9 +175,9 @@ TestFixture::testParse(const string &selection) select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.what()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() != NULL); + ASSERT_TRUE(select.get() != nullptr); } @@ -189,9 +194,9 @@ TestFixture::testParseFail(const string &selection) select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.getMessage().c_str()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() == NULL); + ASSERT_TRUE(select.get() == nullptr); } @@ -208,15 +213,15 @@ TestFixture::testPrune(const string &selection, const string &exp, const string select = parser.parse(selection); } catch (document::select::ParsingFailedException &e) { LOG(info, "Parse failed: %s", e.what()); - select.reset(0); + select.reset(); } - ASSERT_TRUE(select.get() != NULL); + ASSERT_TRUE(select.get() != nullptr); std::ostringstream os; select->print(os, true, ""); LOG(info, "ParseTree: '%s'", os.str().c_str()); const DocumentType *docType = repo.getDocumentType(docTypeName); - ASSERT_TRUE(docType != NULL); - Document::UP emptyDoc(new Document(*docType, document::DocumentId("id:ns:" + docTypeName + "::1"))); + ASSERT_TRUE(docType != nullptr); + auto emptyDoc = std::make_unique<Document>(*docType, document::DocumentId("id:ns:" + docTypeName + "::1")); emptyDoc->setRepo(repo); SelectPruner pruner(docTypeName, &_amgr, *emptyDoc, repo, _hasFields, _hasDocuments); pruner.process(*select); @@ -788,6 +793,29 @@ TEST_F("Test that field values are invalid when disabling document access", Test "test.aa == 4 and test.ae == 5 and invalid"); } +TEST_F("Imported fields with matching attribute names are supported", TestFixture) +{ + f.testPrune("test.my_imported_field > 0", + "test.my_imported_field > 0"); +} + +// Edge case: document type reconfigured but attribute not yet visible in Proton +TEST_F("Imported fields without matching attribute are mapped to constant NullValue", TestFixture) +{ + f.testPrune("test.my_missing_imported_field != test.aa", "null != test.aa"); + // Simplified to -> "null != null" -> "false" + f.testPrune("test.my_missing_imported_field != null", "false"); + // Simplified to -> "null > 0" -> "invalid", as null is not well-defined + // for operators other than (in-)equality. + f.testPrune("test.my_missing_imported_field > 0", "invalid"); +} + +TEST_F("Complex imported field references return Invalid", TestFixture) +{ + f.testPrune("test.my_imported_field.foo", "invalid"); + f.testPrune("test.my_imported_field[123]", "invalid"); + f.testPrune("test.my_imported_field{foo}", "invalid"); +} } // namespace diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 48ab09ffcb2..607a644a302 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -48,43 +48,51 @@ TEST("requireThatAverageTimesAreRecorded") { EXPECT_APPROX(0.0, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(0u, stats.matchTimeCount()); EXPECT_EQUAL(0u, stats.groupingTimeCount()); EXPECT_EQUAL(0u, stats.rerankTimeCount()); EXPECT_EQUAL(0u, stats.queryCollateralTimeCount()); + EXPECT_EQUAL(0u, stats.querySetupTimeCount()); EXPECT_EQUAL(0u, stats.queryLatencyCount()); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyAvg(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.02, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.2, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(4.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(4.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.queryLatencyAvg(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) .queryCollateralTime(10.0) + .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) .queryCollateralTime(10.0).queryCollateralTime(6.0) + .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.03, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeAvg(), 0.00001); EXPECT_APPROX(6.0, stats.queryCollateralTimeAvg(), 0.00001); + EXPECT_APPROX(6.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(4u, stats.matchTimeCount()); EXPECT_EQUAL(4u, stats.groupingTimeCount()); EXPECT_EQUAL(4u, stats.rerankTimeCount()); EXPECT_EQUAL(4u, stats.queryCollateralTimeCount()); + EXPECT_EQUAL(4u, stats.querySetupTimeCount()); EXPECT_EQUAL(4u, stats.queryLatencyCount()); } @@ -94,53 +102,63 @@ TEST("requireThatMinMaxTimesAreRecorded") { EXPECT_APPROX(0.0, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.0, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(0.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMax(), 0.00001); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.01, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMax(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.03, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(6.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(6.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyMax(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) .queryCollateralTime(10.0) + .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) .queryCollateralTime(10.0).queryCollateralTime(6.0) + .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); + EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.05, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(2.5, stats.rerankTimeMax(), 0.00001); EXPECT_APPROX(10.0, stats.queryCollateralTimeMax(), 0.00001); + EXPECT_APPROX(10.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(5.0, stats.queryLatencyMax(), 0.00001); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index d123a5711ac..eb4f1f6dc89 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -397,8 +397,8 @@ writefilter.memorylimit double default = 0.8 ## put and update portion of feed is blocked. writefilter.disklimit double default = 0.8 -## Interval between sampling of disk and memory usage. Default is 60 seconds. -writefilter.sampleinterval double default = 60.0 +## Interval between sampling of disk and memory usage. Default is 20 seconds. +writefilter.sampleinterval double default = 20.0 ## The size of the disk partition (in bytes) on which proton basedir is located. ## If set to 0, the disk size is sampled by looking at the filesystem space info. diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index e39061a8389..02bf4f4d7cc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> @@ -613,4 +614,14 @@ AttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> _importedAttributes = std::move(attributes); } +std::shared_ptr<search::attribute::ReadableAttributeVector> +AttributeManager::readable_attribute_vector(const string& name) const +{ + auto attribute = findAttribute(name); + if (attribute || !_importedAttributes) { + return attribute; + } + return _importedAttributes->get(name); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index cf60a0a41e9..881452c6b3d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -178,6 +178,8 @@ public: void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); } + + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index fd1f95e13ba..aee68457b62 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -231,4 +231,13 @@ FilterAttributeManager::getImportedAttributes() const throw vespalib::IllegalArgumentException("Not implemented"); } +std::shared_ptr<search::attribute::ReadableAttributeVector> +FilterAttributeManager::readable_attribute_vector(const string& name) const +{ + if (acceptAttribute(name)) { + return _mgr->readable_attribute_vector(name); + } + return {}; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 099bb2e84ba..0bd04b8f0a5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -54,6 +54,7 @@ public: ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override; + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index ab3b0588f3f..c516bbfc06c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -4,6 +4,7 @@ #include "selectcontext.h" #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> @@ -31,14 +32,10 @@ using vespalib::make_string; AttributeFieldValueNode:: AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const std::shared_ptr<search::AttributeVector> &attribute) + uint32_t attr_guard_index) : FieldValueNode(doctype, field), - _attribute(attribute) + _attr_guard_index(attr_guard_index) { - const AttributeVector &v(*_attribute); - // Only handle single value attribute vectors for now - assert(v.getCollectionType() == CollectionType::SINGLE); - (void) v; } @@ -46,10 +43,10 @@ std::unique_ptr<document::select::Value> AttributeFieldValueNode:: getValue(const Context &context) const { - const SelectContext &sc(static_cast<const SelectContext &>(context)); + const auto &sc(static_cast<const SelectContext &>(context)); uint32_t docId(sc._docId); assert(docId != 0u); - const AttributeVector &v(*_attribute); + const auto& v = sc.guarded_attribute_at_index(_attr_guard_index); if (v.isUndefined(docId)) { return std::make_unique<NullValue>(); } @@ -86,10 +83,10 @@ getValue(const Context &context) const case BasicType::PREDICATE: case BasicType::TENSOR: case BasicType::REFERENCE: - throw new IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", - v.getName().c_str(), v.getInternalBasicType().asString())); + throw IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection", + v.getName().c_str(), BasicType(v.getBasicType()).asString())); case BasicType::MAX_TYPE: - throw new IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType())); + throw IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType())); } return std::make_unique<NullValue>();; @@ -106,7 +103,7 @@ AttributeFieldValueNode::traceValue(const Context &context, std::ostream& out) c document::select::ValueNode::UP AttributeFieldValueNode::clone() const { - return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute)); + return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attr_guard_index)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h index 9ac6ce0e32b..89d1dac321b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h @@ -3,18 +3,19 @@ #include <vespa/document/select/valuenodes.h> -namespace search { class AttributeVector; } +namespace search { class ReadableAttributeVector; } namespace proton { class AttributeFieldValueNode : public document::select::FieldValueNode { using Context = document::select::Context; - std::shared_ptr<search::AttributeVector> _attribute; + uint32_t _attr_guard_index; public: + // Precondition: attribute must be of a single-value type. AttributeFieldValueNode(const vespalib::string& doctype, const vespalib::string& field, - const std::shared_ptr<search::AttributeVector> &attribute); + uint32_t attr_guard_index); std::unique_ptr<document::select::Value> getValue(const Context &context) const override; std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override; diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index d007b030e6b..21f757fba79 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -7,6 +7,7 @@ #include "selectpruner.h" #include <vespa/document/select/parser.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/iattributemanager.h> namespace proton { @@ -24,7 +25,7 @@ namespace { class AttrVisitor : public document::select::CloningVisitor { public: - typedef std::map<vespalib::string, uint32_t> AttrMap; + using AttrMap = std::map<vespalib::string, uint32_t>; AttrMap _amap; const search::IAttributeManager &_amgr; @@ -62,7 +63,7 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At AttrVisitor::~AttrVisitor() = default; -bool isSingleValueThatWEHandle(BasicType type) { +bool isSingleValueThatWeHandle(BasicType type) { return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE); } @@ -75,17 +76,18 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) bool complex = false; vespalib::string name = SelectUtils::extractFieldName(expr, complex); - AttributeGuard::UP ag(_amgr.getAttribute(name)); - if (ag->valid()) { + auto av = _amgr.readable_attribute_vector(name); + if (av) { if (complex) { ++_complexAttrs; // Don't try to optimize complex attribute references yet. _valueNode = expr.clone(); return; } - std::shared_ptr<search::AttributeVector> av(ag->getSP()); - if (av->getCollectionType() == CollectionType::SINGLE) { - if (isSingleValueThatWEHandle(av->getBasicType())) { + auto guard = av->makeReadGuard(false); + const auto* attr = guard->attribute(); + if (attr->getCollectionType() == CollectionType::SINGLE) { + if (isSingleValueThatWeHandle(attr->getBasicType())) { ++_svAttrs; auto it(_amap.find(name)); uint32_t idx(invalidIdx()); @@ -99,7 +101,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr) idx = it->second; } assert(idx != invalidIdx()); - _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, av); + _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, idx); } else { ++_complexAttrs; // Don't try to optimize predicate/tensor/reference attributes yet. diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h index e9eb452889e..563eed75c32 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h @@ -15,6 +15,8 @@ namespace search { class IAttributeManager; } +namespace search::attribute { class ReadableAttributeVector; } + namespace proton { class SelectContext; @@ -44,7 +46,7 @@ public: const document::select::Node &selectNode() const; }; - using AttributeVectors = std::vector<std::shared_ptr<search::AttributeVector>>; + using AttributeVectors = std::vector<std::shared_ptr<search::attribute::ReadableAttributeVector>>; private: // Single value attributes referenced from selection expression diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp index 3275fbe06d4..ee713e19385 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp @@ -3,20 +3,22 @@ #include "selectcontext.h" #include "cachedselect.h" #include <vespa/document/select/value.h> -#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/readable_attribute_vector.h> +#include <cassert> namespace proton { using document::select::Value; using document::select::Context; -using search::AttributeGuard; using search::AttributeVector; +using search::attribute::AttributeReadGuard; namespace select { - struct Guards : public std::vector<AttributeGuard> { - using Parent = std::vector<AttributeGuard>; - using Parent::Parent; - }; +struct Guards : public std::vector<std::unique_ptr<AttributeReadGuard>> { + using Parent = std::vector<std::unique_ptr<AttributeReadGuard>>; + using Parent::Parent; +}; } SelectContext::SelectContext(const CachedSelect &cachedSelect) @@ -26,23 +28,30 @@ SelectContext::SelectContext(const CachedSelect &cachedSelect) _cachedSelect(cachedSelect) { } -SelectContext::~SelectContext() { } +SelectContext::~SelectContext() = default; void SelectContext::getAttributeGuards() { - _guards->resize(_cachedSelect.attributes().size()); - auto j(_cachedSelect.attributes().begin()); - for (std::vector<AttributeGuard>::iterator i(_guards->begin()), ie(_guards->end()); i != ie; ++i, ++j) { - *i = AttributeGuard(*j); + _guards->clear(); + _guards->reserve(_cachedSelect.attributes().size()); + for (const auto& attr : _cachedSelect.attributes()) { + _guards->emplace_back(attr->makeReadGuard(false)); } } - void SelectContext::dropAttributeGuards() { _guards->clear(); } +const search::attribute::IAttributeVector& +SelectContext::guarded_attribute_at_index(uint32_t index) const noexcept +{ + assert(index < _guards->size()); + assert((*_guards)[index].get() != nullptr); + return *((*_guards)[index])->attribute(); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h index d37c6eb6f14..4dd9f873088 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h @@ -3,6 +3,8 @@ #include <vespa/document/select/context.h> +namespace search::attribute { class IAttributeVector; } + namespace proton { class CachedSelect; @@ -19,6 +21,8 @@ public: void dropAttributeGuards(); uint32_t _docId; + + const search::attribute::IAttributeVector& guarded_attribute_at_index(uint32_t index) const noexcept; private: std::unique_ptr<select::Guards> _guards; const CachedSelect &_cachedSelect; diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp index 6cad54f134b..2c237074907 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp @@ -12,6 +12,7 @@ #include <vespa/document/select/invalidconstant.h> #include <vespa/document/select/valuenodes.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/iattributemanager.h> using document::select::And; @@ -97,10 +98,7 @@ SelectPruner::SelectPruner(const SelectPruner *rhs) } -SelectPruner::~SelectPruner() -{ -} - +SelectPruner::~SelectPruner() = default; void SelectPruner::visitAndBranch(const And &expr) @@ -395,7 +393,6 @@ SelectPruner::visitIdValueNode(const IdValueNode &expr) CloningVisitor::visitIdValueNode(expr); } - void SelectPruner::visitFieldValueNode(const FieldValueNode &expr) { @@ -406,32 +403,33 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) const document::DocumentType *docType = _repo.getDocumentType(_docType); bool complex = false; // Cannot handle attribute if complex expression vespalib::string name = SelectUtils::extractFieldName(expr, complex); - try { - std::unique_ptr<Field> fp(new Field(docType->getField(name))); - if (!fp) { + const bool is_imported = docType->has_imported_field_name(name); + if (complex || !is_imported) { + try { + std::unique_ptr<Field> fp(new Field(docType->getField(name))); + if (!fp) { + setInvalidVal(); + return; + } + } catch (FieldNotFoundException &) { + setInvalidVal(); + return; + } + try { + FieldPath path; + docType->buildFieldPath(path, expr.getFieldName()); + } catch (vespalib::IllegalArgumentException &) { + setInvalidVal(); + return; + } catch (FieldNotFoundException &) { setInvalidVal(); return; } - } catch (FieldNotFoundException &) { - setInvalidVal(); - return; - } - try { - FieldPath path; - docType->buildFieldPath(path, expr.getFieldName()); - } catch (vespalib::IllegalArgumentException &) { - setInvalidVal(); - return; - } catch (FieldNotFoundException &) { - setInvalidVal(); - return; } _constVal = false; if (!_hasFields) { // If we're working on removed document sub db then we have no fields. - _constVal = true; - _valueNode.reset(new NullValueNode()); - _priority = NullValPriority; + set_null_value_node(); return; } @@ -440,13 +438,18 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr) bool svAttr = false; bool attrField = false; if (_amgr != nullptr) { - AttributeGuard::UP ag(_amgr->getAttribute(name)); - if (ag->valid()) { + auto attr = _amgr->readable_attribute_vector(name); + if (attr) { attrField = true; - auto av(ag->getSP()); - if (av->getCollectionType() == CollectionType::SINGLE && !complex) { + auto ag = attr->makeReadGuard(false); + if ((ag->attribute()->getCollectionType() == CollectionType::SINGLE) && !complex) { svAttr = true; } + } else if (is_imported) { + // Imported field present in document config but not yet in attribute config. + // Treat as missing (null) in document, as this matches behavior elsewhere in the pipeline. + set_null_value_node(); + return; } } if (!_hasDocuments && !svAttr) { @@ -538,6 +541,13 @@ SelectPruner::setInvalidConst() _node.reset(new InvalidConstant("invalid")); } +void +SelectPruner::set_null_value_node() +{ + _constVal = true; + _valueNode = std::make_unique<NullValueNode>(); + _priority = NullValPriority; +} void SelectPruner::setTernaryConst(bool val) diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h index 4350247d8b3..2c3729006c3 100644 --- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h +++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h @@ -81,6 +81,7 @@ private: void setInvalidVal(); void setInvalidConst(); void setTernaryConst(bool val); + void set_null_value_node(); void resolveTernaryConst(bool wantInverted); bool isInvalidVal() const; bool isNullVal() const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 6c97dc8c9ef..630ac66f4f1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -287,7 +287,9 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), estHits, reply->totalHitCount, request.ranking.c_str()); } - my_stats.queryCollateralTime(vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg()); + double querySetupTime = vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg(); + my_stats.queryCollateralTime(querySetupTime); // TODO: Remove in Vespa 8 + my_stats.querySetupTime(querySetupTime); { vespalib::duration duration = request.getTimeUsed(); std::lock_guard<std::mutex> guard(_statsLock); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index fff8c94c8be..8cc98816241 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -29,7 +29,8 @@ MatchingStats::MatchingStats() _softDoomed(0), _doomOvertime(), _softDoomFactor(INITIAL_SOFT_DOOM_FACTOR), - _queryCollateralTime(), + _queryCollateralTime(), // TODO: Remove in Vespa 8 + _querySetupTime(), _queryLatency(), _matchTime(), _groupingTime(), @@ -69,7 +70,8 @@ MatchingStats::add(const MatchingStats &rhs) _softDoomed += rhs.softDoomed(); _doomOvertime.add(rhs._doomOvertime); - _queryCollateralTime.add(rhs._queryCollateralTime); + _queryCollateralTime.add(rhs._queryCollateralTime); // TODO: Remove in Vespa 8 + _querySetupTime.add(rhs._querySetupTime); _queryLatency.add(rhs._queryLatency); _matchTime.add(rhs._matchTime); _groupingTime.add(rhs._groupingTime); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index f5eccdd1127..6b533aab7e5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -125,7 +125,8 @@ private: size_t _softDoomed; Avg _doomOvertime; double _softDoomFactor; - Avg _queryCollateralTime; + Avg _queryCollateralTime; // TODO: Remove in Vespa 8 + Avg _querySetupTime; Avg _queryLatency; Avg _matchTime; Avg _groupingTime; @@ -168,12 +169,19 @@ public: double softDoomFactor() const { return _softDoomFactor; } MatchingStats &updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::duration softLimit, vespalib::duration duration); + // TODO: Remove in Vespa 8 MatchingStats &queryCollateralTime(double time_s) { _queryCollateralTime.set(time_s); return *this; } double queryCollateralTimeAvg() const { return _queryCollateralTime.avg(); } size_t queryCollateralTimeCount() const { return _queryCollateralTime.count(); } double queryCollateralTimeMin() const { return _queryCollateralTime.min(); } double queryCollateralTimeMax() const { return _queryCollateralTime.max(); } + MatchingStats &querySetupTime(double time_s) { _querySetupTime.set(time_s); return *this; } + double querySetupTimeAvg() const { return _querySetupTime.avg(); } + size_t querySetupTimeCount() const { return _querySetupTime.count(); } + double querySetupTimeMin() const { return _querySetupTime.min(); } + double querySetupTimeMax() const { return _querySetupTime.max(); } + MatchingStats &queryLatency(double time_s) { _queryLatency.set(time_s); return *this; } double queryLatencyAvg() const { return _queryLatency.avg(); } size_t queryLatencyCount() const { return _queryLatency.count(); } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index f4b8203f8e2..05370920354 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -119,6 +119,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::update(const MatchingStats &stats) queries.inc(stats.queries()); queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); + querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), + stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), stats.queryLatencyMin(), stats.queryLatencyMax()); } @@ -131,6 +133,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent) queries("queries", {}, "Number of queries executed", this), softDoomedQueries("soft_doomed_queries", {}, "Number of queries hitting the soft timeout", this), queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), + querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { } @@ -152,6 +155,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics groupingTime("grouping_time", {}, "Average time (sec) spent on grouping", this), rerankTime("rerank_time", {}, "Average time (sec) spent on 2nd phase ranking", this), queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), + querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { softDoomFactor.set(MatchingStats::INITIAL_SOFT_DOOM_FACTOR); @@ -204,6 +208,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const Match stats.rerankTimeMin(), stats.rerankTimeMax()); queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); + querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), + stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), stats.queryLatencyMin(), stats.queryLatencyMax()); if (stats.getNumPartitions() > 0) { diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 01ba271a08f..26dd52a8577 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -116,6 +116,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::LongCountMetric queries; metrics::LongCountMetric softDoomedQueries; metrics::DoubleAverageMetric queryCollateralTime; + metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; struct RankProfileMetrics : metrics::MetricSet { @@ -145,6 +146,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::DoubleAverageMetric groupingTime; metrics::DoubleAverageMetric rerankTime; metrics::DoubleAverageMetric queryCollateralTime; + metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; DocIdPartitions partitions; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 9445a0a5206..1a6fc6cdbfd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -163,7 +163,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) _listeners() { } -DiskMemUsageFilter::~DiskMemUsageFilter() { } +DiskMemUsageFilter::~DiskMemUsageFilter() = default; void DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in) diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index 40c74808e72..b05cc261728 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -15,11 +15,7 @@ class DiskMemUsageState ResourceUsageState _memoryState; public: - DiskMemUsageState() - : _diskState(), - _memoryState() - { - } + DiskMemUsageState() = default; DiskMemUsageState(const ResourceUsageState &diskState_, const ResourceUsageState &memoryState_) : _diskState(diskState_), diff --git a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h index c70a6c502f1..63ff269580e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h @@ -36,7 +36,7 @@ public: */ virtual void unBlock(BlockedReason reason) = 0; - virtual IBlockableMaintenanceJob *asBlockable() override { return this; } + IBlockableMaintenanceJob *asBlockable() override { return this; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h index 6d0739e1aed..df6889ecf04 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h @@ -31,7 +31,7 @@ public: _interval(interval) {} - virtual ~IMaintenanceJob() {} + virtual ~IMaintenanceJob() = default; virtual const vespalib::string &getName() const { return _name; } virtual vespalib::duration getDelay() const { return _delay; } diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp index 6c0c0863fe1..e535b05393c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp @@ -41,9 +41,7 @@ MoveOperationLimiter::MoveOperationLimiter(IBlockableMaintenanceJob *job, { } -MoveOperationLimiter::~MoveOperationLimiter() -{ -} +MoveOperationLimiter::~MoveOperationLimiter() = default; void MoveOperationLimiter::clearJob() diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 87fa2053508..57c86855217 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -78,6 +78,9 @@ public: void asyncForAttribute(const vespalib::string & name, std::unique_ptr<IAttributeFunctor> func) const override { _mock.asyncForAttribute(name, std::move(func)); } + std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override { + return _mock.readable_attribute_vector(name); + } }; } diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 2c8eff4f4ad..d754fd78394 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -210,6 +210,7 @@ vespa_define_module( src/tests/sortspec src/tests/stringenum src/tests/tensor/dense_tensor_store + src/tests/tensor/hnsw_index src/tests/transactionlog src/tests/transactionlogstress src/tests/true diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 62cde4d2c9c..7d09b2aa0b8 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -34,6 +34,7 @@ private: void testGuards(); void testConfigConvert(); void testContext(); + void can_get_readable_attribute_vector_by_name(); bool assertDataType(BT::Type exp, @@ -377,6 +378,21 @@ AttributeManagerTest::testContext() } } +void +AttributeManagerTest::can_get_readable_attribute_vector_by_name() +{ + auto attr = AttributeFactory::createAttribute("cool_attr", Config(BT::INT32, CT::SINGLE)); + // Ensure there's something to actually load, or fetching the attribute will throw. + attr->addDocs(64); + attr->commit(); + AttributeManager manager; + manager.add(attr); + auto av = manager.readable_attribute_vector("cool_attr"); + EXPECT_EQUAL(av.get(), static_cast<ReadableAttributeVector*>(attr.get())); + av = manager.readable_attribute_vector("uncool_attr"); + EXPECT_TRUE(av.get() == nullptr); +} + int AttributeManagerTest::Main() { TEST_INIT("attributemanager_test"); @@ -385,6 +401,7 @@ int AttributeManagerTest::Main() testGuards(); testConfigConvert(); testContext(); + can_get_readable_attribute_vector_by_name(); TEST_DONE(); } diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index cfa05b7a765..48a06bec9e8 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -219,6 +219,15 @@ TEST_F("Weighted floating point attribute values can be retrieved via reference" assert_multi_value_matches<WeightedFloat>(f, DocId(3), doc7_values); } +TEST_F("isUndefined() works for primitive attribute type", Fixture) { + reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>( + f, BasicType::INT32, + {{DocId(3), dummy_gid(7), DocId(7), 5678}}); + + EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(3))); // Mapped + EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(2))); // Not mapped +} + struct SingleStringAttrFixture : Fixture { SingleStringAttrFixture() : Fixture() { setup(); @@ -255,6 +264,11 @@ TEST_F("findEnum() returns target vector enum via reference", SingleStringAttrFi EXPECT_EQUAL(expected_handle, actual_handle); } +TEST_F("isUndefined() works for enumerated attribute type", SingleStringAttrFixture) { + EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(2))); // Mapped + EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(3))); // Not mapped +} + // Note: assumes that fixture has set up a string enum of value "foo" in target attribute template <typename FixtureType> void verify_get_string_from_enum_is_mapped(FixtureType& f) { diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 4818287b429..2eafeab20bd 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -123,6 +123,15 @@ public: return IAttributeContext::UP(); } + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override { + if (name == field) { + return _attribute_vector; + } else if (name == other) { + return _other; + } + return {}; + } + void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; }; diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index e9addae07b9..24be21f65ec 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -100,6 +100,10 @@ public: void asyncForAttribute(const vespalib::string &, std::unique_ptr<IAttributeFunctor>) const override { assert(!"Not implemented"); } + + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override { + return _attribute_vector; + } }; constexpr uint32_t DOCID_LIMIT = 3; diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt new file mode 100644 index 00000000000..04c7312a63f --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_hnsw_index_test_app TEST + SOURCES + hnsw_index_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app) diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp new file mode 100644 index 00000000000..634569cbc8c --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -0,0 +1,236 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/searchlib/tensor/doc_vector_access.h> +#include <vespa/searchlib/tensor/hnsw_index.h> +#include <vespa/searchlib/tensor/random_level_generator.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vector> + +#include <vespa/log/log.h> +LOG_SETUP("hnsw_index_test"); + +using namespace search::tensor; + +template <typename FloatType> +class MyDocVectorAccess : public DocVectorAccess { +private: + using Vector = std::vector<FloatType>; + using ArrayRef = vespalib::ConstArrayRef<FloatType>; + std::vector<Vector> _vectors; + +public: + MyDocVectorAccess() : _vectors() {} + MyDocVectorAccess& set(uint32_t docid, const Vector& vec) { + if (docid >= _vectors.size()) { + _vectors.resize(docid + 1); + } + _vectors[docid] = vec; + return *this; + } + vespalib::tensor::TypedCells get_vector(uint32_t docid) const override { + ArrayRef ref(_vectors[docid]); + return vespalib::tensor::TypedCells(ref); + } +}; + +struct LevelGenerator : public RandomLevelGenerator { + uint32_t level; + LevelGenerator() : level(0) {} + uint32_t max_level() override { return level; } +}; + +using FloatVectors = MyDocVectorAccess<float>; +using FloatIndex = HnswIndex<float>; +using FloatIndexUP = std::unique_ptr<FloatIndex>; + +class HnswIndexTest : public ::testing::Test { +public: + FloatVectors vectors; + LevelGenerator level_generator; + FloatIndexUP index; + + HnswIndexTest() + : vectors(), + level_generator(), + index() + { + vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3}) + .set(4, {1, 2}).set(5, {8, 3}).set(6, {7, 2}) + .set(7, {3, 5}); + } + void init(bool heuristic_select_neighbors) { + index = std::make_unique<FloatIndex>(vectors, level_generator, + HnswIndexBase::Config(2, 1, 10, heuristic_select_neighbors)); + } + void add_document(uint32_t docid, uint32_t max_level = 0) { + level_generator.level = max_level; + index->add_document(docid); + } + void remove_document(uint32_t docid) { + index->remove_document(docid); + } + void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) { + EXPECT_EQ(exp_docid, index->get_entry_docid()); + EXPECT_EQ(exp_level, index->get_entry_level()); + } + void expect_level_0(uint32_t docid, const HnswNode::LinkArray& exp_links) { + auto node = index->get_node(docid); + ASSERT_EQ(1, node.size()); + EXPECT_EQ(exp_links, node.level(0)); + } + void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) { + auto act_node = index->get_node(docid); + ASSERT_EQ(exp_levels.size(), act_node.size()); + EXPECT_EQ(exp_levels, act_node.levels()); + } +}; + + +TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_neighbors) +{ + init(false); + + add_document(1); + expect_level_0(1, {}); + + add_document(2); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + + add_document(3); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2}); + + add_document(4); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2, 4}); + expect_level_0(4, {1, 3}); + + add_document(5); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5}); + expect_level_0(3, {1, 2, 4, 5}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3}); + + add_document(6); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_level_0(3, {1, 2, 4, 5}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3, 6}); + expect_level_0(6, {2, 5}); + + add_document(7); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6, 7}); + expect_level_0(3, {1, 2, 4, 5, 7}); + expect_level_0(4, {1, 3}); + expect_level_0(5, {2, 3, 6}); + expect_level_0(6, {2, 5}); + expect_level_0(7, {2, 3}); +} + +TEST_F(HnswIndexTest, 2d_vectors_inserted_and_removed) +{ + init(false); + + add_document(1); + expect_level_0(1, {}); + expect_entry_point(1, 0); + + add_document(2); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + expect_entry_point(1, 0); + + add_document(3); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_level_0(3, {1, 2}); + expect_entry_point(1, 0); + + remove_document(2); + expect_level_0(1, {3}); + expect_level_0(3, {1}); + expect_entry_point(1, 0); + + remove_document(1); + expect_level_0(3, {}); + expect_entry_point(3, 0); + + remove_document(3); + expect_entry_point(0, -1); +} + +TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_select_neighbors) +{ + init(true); + + add_document(1); + expect_entry_point(1, 0); + expect_level_0(1, {}); + + add_document(2); + expect_entry_point(1, 0); + expect_level_0(1, {2}); + expect_level_0(2, {1}); + + // Doc 3 is also added to level 1 + add_document(3, 1); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3}); + expect_level_0(2, {1, 3}); + expect_levels(3, {{1, 2}, {}}); + + // Doc 4 is closest to 1 and they are linked. + // Doc 4 is NOT linked to 3 as the distance between 4 and 3 is greater than the distance between 3 and 1. + // Doc 3 is therefore reachable via 1. Same argument for why doc 4 is not linked to 2. + add_document(4); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3}); + expect_levels(3, {{1, 2}, {}}); + expect_level_0(4, {1}); + + // Doc 5 is closest to 2 and they are linked. + // The other docs are reachable via 2, and no other links are created. Same argument as with doc 4 above. + add_document(5); + expect_entry_point(3, 1); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5}); + expect_levels(3, {{1, 2}, {}}); + expect_level_0(4, {1}); + expect_level_0(5, {2}); + + // Doc 6 is closest to 5 and they are linked. + // Doc 6 is also linked to 2 as the distance between 6 and 2 is less than the distance between 2 and 5. + // Doc 6 is also added to level 1 and 2, and linked to doc 3 in level 1. + add_document(6, 2); + expect_entry_point(6, 2); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_levels(3, {{1, 2}, {6}}); + expect_level_0(4, {1}); + expect_level_0(5, {2, 6}); + expect_levels(6, {{2, 5}, {3}, {}}); + + // Doc 7 is closest to 3 and they are linked. + // Doc 7 is also linked to 6 as the distance between 7 and 6 is less than the distance between 6 and 3. + // Docs 1, 2, 4 are reachable via 3. + add_document(7); + expect_entry_point(6, 2); + expect_level_0(1, {2, 3, 4}); + expect_level_0(2, {1, 3, 5, 6}); + expect_levels(3, {{1, 2, 7}, {6}}); + expect_level_0(4, {1}); + expect_level_0(5, {2, 6}); + expect_levels(6, {{2, 5, 7}, {3}, {}}); + expect_level_0(7, {3, 6}); +} + +GTEST_MAIN_RUN_ALL_TESTS() + diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index da280aa7bda..5fa32e8ad75 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -131,7 +131,7 @@ bool AttributeManager::hasReaders() const const AttributeManager::VectorHolder * AttributeManager::findAndLoadAttribute(const string & name) const { - const VectorHolder * loadedVector(NULL); + const VectorHolder * loadedVector(nullptr); AttributeMap::const_iterator found = _attributes.find(name); if (found != _attributes.end()) { AttributeVector & vec = *found->second; @@ -263,5 +263,14 @@ AttributeManager::asyncForAttribute(const vespalib::string &, std::unique_ptr<at throw std::runtime_error("search::AttributeManager::asyncForAttribute should never be called."); } +std::shared_ptr<attribute::ReadableAttributeVector> +AttributeManager::readable_attribute_vector(const string& name) const +{ + const auto* attr = findAndLoadAttribute(name); + if (attr) { + return *attr; + } + return {}; +} } diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h index 1c60ab00585..9f30ff87a70 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h @@ -49,6 +49,8 @@ public: void getAttributeList(AttributeList & list) const override; attribute::IAttributeContext::UP createContext() const override; + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; + const Snapshot & getSnapshot() const { return _snapShot; } const string & getBaseDir() const { return _baseDir; } void setBaseDir(const string & base); diff --git a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h index ef9403e1c4b..51e432f2035 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h @@ -8,7 +8,10 @@ namespace search { -namespace attribute { class AttributeReadGuard; } +namespace attribute { +class AttributeReadGuard; +class ReadableAttributeVector; +} /** * This is an interface used to access all registered attribute vectors. @@ -17,12 +20,17 @@ class IAttributeManager : public attribute::IAttributeExecutor { public: IAttributeManager(const IAttributeManager &) = delete; IAttributeManager & operator = (const IAttributeManager &) = delete; - typedef std::shared_ptr<IAttributeManager> SP; - typedef vespalib::string string; + using SP = std::shared_ptr<IAttributeManager>; + using string = vespalib::string; /** * Returns a view of the attribute vector with the given name. * + * NOTE: this method is deprecated! Prefer using readable_attribute_vector(name) instead, + * as that enforces appropriate guards to be taken before accessing the underlying vector. + * + * TODO remove this when all usages are gone. + * * @param name name of the attribute vector. * @return view of the attribute vector or empty view if the attribute vector does not exists. **/ @@ -52,9 +60,15 @@ public: virtual attribute::IAttributeContext::UP createContext() const = 0; /** - * Virtual destructor to allow safe subclassing. - **/ - virtual ~IAttributeManager() {} + * Looks up and returns a readable attribute vector shared_ptr with the provided name. + * This transparently supports imported attribute vectors. + * + * @param name name of the attribute vector. + * @return The attribute vector, or an empty shared_ptr if no vector was found with the given name. + */ + virtual std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const = 0; + + ~IAttributeManager() override = default; protected: IAttributeManager() = default; }; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index b097b27ea53..f2949119ae2 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -152,6 +152,10 @@ bool ImportedAttributeVectorReadGuard::isImported() const return true; } +bool ImportedAttributeVectorReadGuard::isUndefined(DocId doc) const { + return _target_attribute.isUndefined(getTargetLid(doc)); +} + long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc, void *serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index f130a095006..b31b11b8c74 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -81,6 +81,7 @@ public: bool getIsFastSearch() const override; uint32_t getCommittedDocIdLimit() const override; bool isImported() const override; + bool isUndefined(DocId doc) const override; protected: long onSerializeForAscendingSort(DocId doc, void * serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 3dcc0df477d..07f8077ae97 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -42,9 +42,9 @@ template <typename EntryT, typename RefT> void MultiValueMapping<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values) { - ConstArrayRef oldValues = _store.get(_indices[docId]); + auto oldValues = _store.get_writable(_indices[docId]); assert(oldValues.size() == values.size()); - EntryT *dst = const_cast<EntryT *>(&oldValues[0]); + EntryT *dst = &oldValues[0]; for (auto &src : values) { *dst = src; ++dst; diff --git a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h index 03058362cb1..0e3954bdcb5 100644 --- a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h +++ b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h @@ -13,7 +13,7 @@ class AttributeReadGuard; */ class ReadableAttributeVector { public: - virtual ~ReadableAttributeVector() {} + virtual ~ReadableAttributeVector() = default; virtual std::unique_ptr<AttributeReadGuard> makeReadGuard(bool stableEnumGuard) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index c7976fd15ae..847cd3623c3 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -79,7 +79,7 @@ Chunk::Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc) : os >> _lastSerial; } -Chunk::~Chunk() { } +Chunk::~Chunk() = default; vespalib::ConstBufferRef Chunk::getLid(uint32_t lid) const diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 70295d81a93..d12a3a54a93 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -203,6 +203,13 @@ seek_past(LidInfoWithLidV::const_iterator begin, LidInfoWithLidV::const_iterator return begin; } +struct LidAndBuffer { + LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) : _lid(lid), _size(sz), _buf(std::move(buf)) {} + uint32_t _lid; + uint32_t _size; + vespalib::alloc::Alloc _buf; +}; + } void @@ -211,6 +218,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB if (count == 0) { return; } if (!frozen()) { vespalib::hash_map<uint32_t, ChunkInfo> chunksOnFile; + std::vector<LidAndBuffer> buffers; { LockGuard guard(_lock); for (size_t i(0); i < count; i++) { @@ -225,12 +233,18 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB assert(chunk == _active->getId()); buffer = _active->getLid(li.getLid()); } - visitor.visit(li.getLid(), buffer); + auto copy = vespalib::alloc::Alloc::alloc(buffer.size()); + memcpy(copy.get(), buffer.data(), buffer.size()); + buffers.emplace_back(li.getLid(), buffer.size(), std::move(copy)); } else { chunksOnFile[chunk] = _chunkInfo[chunk]; } } } + for (auto & entry : buffers) { + visitor.visit(entry._lid, vespalib::ConstBufferRef(entry._buf.get(), entry._size)); + entry._buf = vespalib::alloc::Alloc(); + } for (auto & it : chunksOnFile) { auto first = find_first(begin, it.first); auto last = seek_past(first, begin + count, it.first); diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index b1d3356dd62..519dcf874a2 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -356,6 +356,7 @@ createAttributeExecutor(const IAttributeVector *attribute, const vespalib::strin if ((attribute->getCollectionType() == CollectionType::SINGLE) && (attribute->isIntegerType() || attribute->isFloatingPointType())) { { SingleValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator; if (creator.handle(attribute)) return creator.create(stash); } + { SingleValueExecutorCreator<IntegerAttributeTemplate<int8_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } { SingleValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator; if (creator.handle(attribute)) return creator.create(stash); } } diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index e1e351d0726..bf8b59f100c 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -67,59 +67,59 @@ namespace search::features { void setup_search_features(fef::IBlueprintRegistry & registry) { // Prod features. - registry.addPrototype(Blueprint::SP(new AgeBlueprint())); - registry.addPrototype(Blueprint::SP(new AttributeBlueprint())); - registry.addPrototype(Blueprint::SP(new AttributeMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new Bm25Blueprint())); - registry.addPrototype(Blueprint::SP(new ClosenessBlueprint())); - registry.addPrototype(Blueprint::SP(new DebugAttributeWaitBlueprint())); - registry.addPrototype(Blueprint::SP(new DebugWaitBlueprint())); - registry.addPrototype(Blueprint::SP(new DistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new DistanceToPathBlueprint())); - registry.addPrototype(Blueprint::SP(new DotProductBlueprint())); - registry.addPrototype(Blueprint::SP(new ElementCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ElementSimilarityBlueprint())); - registry.addPrototype(Blueprint::SP(new EuclideanDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldInfoBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldLengthBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new FieldTermMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new FirstPhaseBlueprint())); - registry.addPrototype(Blueprint::SP(new FlowCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ForeachBlueprint())); - registry.addPrototype(Blueprint::SP(new FreshnessBlueprint())); - registry.addPrototype(Blueprint::SP(new ItemRawScoreBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchCountBlueprint())); - registry.addPrototype(Blueprint::SP(new MatchesBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeAttributeMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeDotProductBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeFieldMatchBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new NativeRankBlueprint())); - registry.addPrototype(Blueprint::SP(new NowBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryTermCountBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomNormalBlueprint())); - registry.addPrototype(Blueprint::SP(new RandomNormalStableBlueprint())); - registry.addPrototype(Blueprint::SP(new RawScoreBlueprint())); - registry.addPrototype(Blueprint::SP(new SubqueriesBlueprint)); - registry.addPrototype(Blueprint::SP(new TensorFromLabelsBlueprint())); - registry.addPrototype(Blueprint::SP(new TensorFromWeightedSetBlueprint())); - registry.addPrototype(Blueprint::SP(new TermBlueprint())); - registry.addPrototype(Blueprint::SP(new TermDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new TermInfoBlueprint())); - registry.addPrototype(Blueprint::SP(new TextSimilarityBlueprint())); - registry.addPrototype(Blueprint::SP(new ValueBlueprint())); + registry.addPrototype(std::make_shared<AgeBlueprint>()); + registry.addPrototype(std::make_shared<AttributeBlueprint>()); + registry.addPrototype(std::make_shared<AttributeMatchBlueprint>()); + registry.addPrototype(std::make_shared<Bm25Blueprint>()); + registry.addPrototype(std::make_shared<ClosenessBlueprint>()); + registry.addPrototype(std::make_shared<DebugAttributeWaitBlueprint>()); + registry.addPrototype(std::make_shared<DebugWaitBlueprint>()); + registry.addPrototype(std::make_shared<DistanceBlueprint>()); + registry.addPrototype(std::make_shared<DistanceToPathBlueprint>()); + registry.addPrototype(std::make_shared<DotProductBlueprint>()); + registry.addPrototype(std::make_shared<ElementCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ElementSimilarityBlueprint>()); + registry.addPrototype(std::make_shared<EuclideanDistanceBlueprint>()); + registry.addPrototype(std::make_shared<FieldInfoBlueprint>()); + registry.addPrototype(std::make_shared<FieldLengthBlueprint>()); + registry.addPrototype(std::make_shared<FieldMatchBlueprint>()); + registry.addPrototype(std::make_shared<FieldTermMatchBlueprint>()); + registry.addPrototype(std::make_shared<FirstPhaseBlueprint>()); + registry.addPrototype(std::make_shared<FlowCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ForeachBlueprint>()); + registry.addPrototype(std::make_shared<FreshnessBlueprint>()); + registry.addPrototype(std::make_shared<ItemRawScoreBlueprint>()); + registry.addPrototype(std::make_shared<MatchBlueprint>()); + registry.addPrototype(std::make_shared<MatchCountBlueprint>()); + registry.addPrototype(std::make_shared<MatchesBlueprint>()); + registry.addPrototype(std::make_shared<NativeAttributeMatchBlueprint>()); + registry.addPrototype(std::make_shared<NativeDotProductBlueprint>()); + registry.addPrototype(std::make_shared<NativeFieldMatchBlueprint>()); + registry.addPrototype(std::make_shared<NativeProximityBlueprint>()); + registry.addPrototype(std::make_shared<NativeRankBlueprint>()); + registry.addPrototype(std::make_shared<NowBlueprint>()); + registry.addPrototype(std::make_shared<QueryBlueprint>()); + registry.addPrototype(std::make_shared<QueryTermCountBlueprint>()); + registry.addPrototype(std::make_shared<RandomBlueprint>()); + registry.addPrototype(std::make_shared<RandomNormalBlueprint>()); + registry.addPrototype(std::make_shared<RandomNormalStableBlueprint>()); + registry.addPrototype(std::make_shared<RawScoreBlueprint>()); + registry.addPrototype(std::make_shared<SubqueriesBlueprint>()); + registry.addPrototype(std::make_shared<TensorFromLabelsBlueprint>()); + registry.addPrototype(std::make_shared<TensorFromWeightedSetBlueprint>()); + registry.addPrototype(std::make_shared<TermBlueprint>()); + registry.addPrototype(std::make_shared<TermDistanceBlueprint>()); + registry.addPrototype(std::make_shared<TermInfoBlueprint>()); + registry.addPrototype(std::make_shared<TextSimilarityBlueprint>()); + registry.addPrototype(std::make_shared<ValueBlueprint>()); // Beta features. - registry.addPrototype(Blueprint::SP(new JaroWinklerDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new ProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new QueryCompletenessBlueprint())); - registry.addPrototype(Blueprint::SP(new ReverseProximityBlueprint())); - registry.addPrototype(Blueprint::SP(new TermEditDistanceBlueprint())); - registry.addPrototype(Blueprint::SP(new TermFieldMdBlueprint())); + registry.addPrototype(std::make_shared<JaroWinklerDistanceBlueprint>()); + registry.addPrototype(std::make_shared<ProximityBlueprint>()); + registry.addPrototype(std::make_shared<QueryCompletenessBlueprint>()); + registry.addPrototype(std::make_shared<ReverseProximityBlueprint>()); + registry.addPrototype(std::make_shared<TermEditDistanceBlueprint>()); + registry.addPrototype(std::make_shared<TermFieldMdBlueprint>()); registry.addPrototype(std::make_shared<ConstantBlueprint>()); // Ranking Expression diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp index 95813be1360..357aec3fc71 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp @@ -5,8 +5,7 @@ #include <vespa/log/log.h> LOG_SETUP(".fef.blueprintfactory"); -namespace search { -namespace fef { +namespace search::fef { BlueprintFactory::BlueprintFactory() : _blueprintMap() @@ -20,7 +19,7 @@ BlueprintFactory::addPrototype(Blueprint::SP proto) if (_blueprintMap.find(name) != _blueprintMap.end()) { LOG(warning, "Blueprint prototype overwritten: %s", name.c_str()); } - _blueprintMap[name] = proto; + _blueprintMap[name] = std::move(proto); } void @@ -41,9 +40,7 @@ BlueprintFactory::createBlueprint(const vespalib::string &name) const if (itr == _blueprintMap.end()) { return Blueprint::SP(); } - Blueprint::UP bp = itr->second->createInstance(); - return Blueprint::SP(bp.release()); + return itr->second->createInstance(); } -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h index 221cca38f5f..346294ba5f5 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h @@ -6,8 +6,7 @@ #include "iblueprintregistry.h" #include <map> -namespace search { -namespace fef { +namespace search::fef { /** * This class implements the blueprint repository interface and acts @@ -54,6 +53,4 @@ public: Blueprint::SP createBlueprint(const vespalib::string &name) const; }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index a9e500fc802..cd2cd949a91 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -44,13 +44,13 @@ struct Compiler : public Blueprint::DependencyHandler { ExecutorSpec spec; const FeatureNameParser &parser; Frame(Blueprint::SP blueprint, const FeatureNameParser &parser_in) - : spec(blueprint), parser(parser_in) {} + : spec(std::move(blueprint)), parser(parser_in) {} }; using Stack = std::vector<Frame>; struct FrameGuard { Stack &stack; - FrameGuard(Stack &stack_in) : stack(stack_in) {} + explicit FrameGuard(Stack &stack_in) : stack(stack_in) {} ~FrameGuard() { stack.pop_back(); } }; @@ -101,11 +101,11 @@ struct Compiler : public Blueprint::DependencyHandler { FeatureRef setup_feature(const FeatureNameParser &parser, Accept accept_type) { Blueprint::SP blueprint = factory.createBlueprint(parser.baseName()); - if (blueprint.get() == nullptr) { + if ( ! blueprint) { return failed(parser.featureName(), vespalib::make_string("unknown basename: '%s'", parser.baseName().c_str())); } - resolve_stack.emplace_back(blueprint, parser); + resolve_stack.emplace_back(std::move(blueprint), parser); FrameGuard frame_guard(resolve_stack); self().spec.blueprint->setName(parser.executorName()); self().spec.blueprint->attach_dependency_handler(*this); @@ -172,13 +172,13 @@ struct Compiler : public Blueprint::DependencyHandler { } // namespace search::fef::<unnamed> BlueprintResolver::ExecutorSpec::ExecutorSpec(Blueprint::SP blueprint_in) - : blueprint(blueprint_in), + : blueprint(std::move(blueprint_in)), inputs(), output_types() { } -BlueprintResolver::ExecutorSpec::~ExecutorSpec() { } -BlueprintResolver::~BlueprintResolver() { } +BlueprintResolver::ExecutorSpec::~ExecutorSpec() = default; +BlueprintResolver::~BlueprintResolver() = default; BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv) @@ -194,7 +194,7 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, void BlueprintResolver::addSeed(vespalib::stringref feature) { - _seeds.push_back(feature); + _seeds.emplace_back(feature); } bool diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h index 317263260fe..2edca268214 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h @@ -8,8 +8,7 @@ #include "blueprint.h" #include "feature_type.h" -namespace search { -namespace fef { +namespace search::fef { class BlueprintFactory; class IIndexEnvironment; @@ -147,5 +146,4 @@ public: const FeatureMap &getSeedMap() const; }; -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h index 3b7634d3e93..43ca5a44303 100644 --- a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h +++ b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h @@ -2,8 +2,7 @@ #pragma once -namespace search { -namespace fef { +namespace search::fef { /** * This is an interface used during plugin setup to register blueprint @@ -23,6 +22,4 @@ public: virtual ~IBlueprintRegistry() {} }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.h b/searchlib/src/vespa/searchlib/fef/parametervalidator.h index 6e0fd7199c8..12044905dd0 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.h +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.h @@ -7,8 +7,7 @@ #include "parameter.h" #include "parameterdescriptions.h" -namespace search { -namespace fef { +namespace search::fef { /** * This class is a validator for a string parameter list given an index environment and a set of parameter descriptions. @@ -83,6 +82,4 @@ public: Result validate(); }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 80729d90bf5..d39be693806 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -153,7 +153,7 @@ RankProgram::resolve(const BlueprintResolver::FeatureMap &features, bool unbox_s } RankProgram::RankProgram(BlueprintResolver::SP resolver) - : _resolver(resolver), + : _resolver(std::move(resolver)), _hot_stash(32768), _cold_stash(), _executors(), @@ -162,7 +162,7 @@ RankProgram::RankProgram(BlueprintResolver::SP resolver) { } -RankProgram::~RankProgram() {} +RankProgram::~RankProgram() = default; void RankProgram::setup(const MatchData &md, diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 440ea3e0ea1..ee88be8ad00 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -27,10 +27,10 @@ using namespace indexproperties; RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv) : _factory(factory), _indexEnv(indexEnv), - _first_phase_resolver(new BlueprintResolver(factory, indexEnv)), - _second_phase_resolver(new BlueprintResolver(factory, indexEnv)), - _summary_resolver(new BlueprintResolver(factory, indexEnv)), - _dumpResolver(new BlueprintResolver(factory, indexEnv)), + _first_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _second_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _summary_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), + _dumpResolver(std::make_shared<BlueprintResolver>(factory, indexEnv)), _firstPhaseRankFeature(), _secondPhaseRankFeature(), _degradationAttribute(), diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index ea5d30d9a47..7c05304d451 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -6,6 +6,8 @@ vespa_add_library(searchlib_tensor OBJECT dense_tensor_store.cpp generic_tensor_attribute.cpp generic_tensor_store.cpp + hnsw_index_base.cpp + hnsw_index.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp tensor_attribute.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h new file mode 100644 index 00000000000..e5dfa35a529 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <cstdint> + +namespace search::tensor { + +/** + * Interface that provides access to the vector that is associated with the the given document id. + * + * All vectors should be the same size and either of type float or double. + */ +class DocVectorAccess { +public: + virtual ~DocVectorAccess() {} + virtual vespalib::tensor::TypedCells get_vector(uint32_t docid) const = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp new file mode 100644 index 00000000000..345f7f551a6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -0,0 +1,167 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_index.h" +#include <vespa/vespalib/util/rcuvector.hpp> + +namespace search::tensor { + +template <typename FloatType> +double +HnswIndex<FloatType>::calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const +{ + auto lhs = get_vector(lhs_docid); + return calc_distance(lhs, rhs_docid); +} + +template <typename FloatType> +double +HnswIndex<FloatType>::calc_distance(const Vector& lhs, uint32_t rhs_docid) const +{ + // TODO: Make it possible to specify the distance function from the outside and make it hardware optimized. + auto rhs = get_vector(rhs_docid); + double result = 0.0; + size_t sz = lhs.size(); + assert(sz == rhs.size()); + for (size_t i = 0; i < sz; ++i) { + double diff = lhs[i] - rhs[i]; + result += diff * diff; + } + return result; +} + +template <typename FloatType> +HnswCandidate +HnswIndex<FloatType>::find_nearest_in_layer(const Vector& input, const HnswCandidate& entry_point, uint32_t level) +{ + HnswCandidate nearest = entry_point; + bool keep_searching = true; + while (keep_searching) { + keep_searching = false; + for (uint32_t neighbor_docid : get_link_array(nearest.docid, level)) { + double dist = calc_distance(input, neighbor_docid); + if (dist < nearest.distance) { + nearest = HnswCandidate(neighbor_docid, dist); + keep_searching = true; + } + } + } + return nearest; +} + +template <typename FloatType> +void +HnswIndex<FloatType>::search_layer(const Vector& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) +{ + NearestPriQ candidates; + // TODO: Add proper handling of visited set. + auto visited = BitVector::create(_node_refs.size()); + for (const auto &entry : best_neighbors.peek()) { + candidates.push(entry); + visited->setBit(entry.docid); + } + double limit_dist = std::numeric_limits<double>::max(); + + while (!candidates.empty()) { + auto cand = candidates.top(); + if (cand.distance > limit_dist) { + break; + } + candidates.pop(); + for (uint32_t neighbor_docid : get_link_array(cand.docid, level)) { + if (visited->testBit(neighbor_docid)) { + continue; + } + visited->setBit(neighbor_docid); + double dist_to_input = calc_distance(input, neighbor_docid); + if (dist_to_input < limit_dist) { + candidates.emplace(neighbor_docid, dist_to_input); + best_neighbors.emplace(neighbor_docid, dist_to_input); + if (best_neighbors.size() > neighbors_to_find) { + best_neighbors.pop(); + limit_dist = best_neighbors.top().distance; + } + } + } + } +} + +template <typename FloatType> +HnswIndex<FloatType>::HnswIndex(const DocVectorAccess& vectors, RandomLevelGenerator& level_generator, const Config& cfg) + : HnswIndexBase(vectors, level_generator, cfg) +{ +} + +template <typename FloatType> +HnswIndex<FloatType>::~HnswIndex() = default; + +template <typename FloatType> +void +HnswIndex<FloatType>::add_document(uint32_t docid) +{ + auto input = get_vector(docid); + _node_refs.ensure_size(docid + 1, AtomicEntryRef()); + // A document cannot be added twice. + assert(!_node_refs[docid].load_acquire().valid()); + int level = make_node_for_document(docid); + if (_entry_docid == 0) { + _entry_docid = docid; + _entry_level = level; + return; + } + + int search_level = _entry_level; + double entry_dist = calc_distance(input, _entry_docid); + HnswCandidate entry_point(_entry_docid, entry_dist); + while (search_level > level) { + entry_point = find_nearest_in_layer(input, entry_point, search_level); + --search_level; + } + + FurthestPriQ best_neighbors; + best_neighbors.push(entry_point); + search_level = std::min(level, _entry_level); + + // Insert the added document in each level it should exist in. + while (search_level >= 0) { + // TODO: Rename to search_level? + search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); + auto neighbors = select_neighbors(best_neighbors.peek(), max_links_for_level(search_level)); + connect_new_node(docid, neighbors, search_level); + // TODO: Shrink neighbors if needed + --search_level; + } + if (level > _entry_level) { + _entry_docid = docid; + _entry_level = level; + } +} + +template <typename FloatType> +void +HnswIndex<FloatType>::remove_document(uint32_t docid) +{ + bool need_new_entrypoint = (docid == _entry_docid); + LinkArray empty; + LevelArrayRef node_levels = get_level_array(docid); + for (int level = node_levels.size(); level-- > 0; ) { + LinkArrayRef my_links = get_link_array(docid, level); + for (uint32_t neighbor_id : my_links) { + if (need_new_entrypoint) { + _entry_docid = neighbor_id; + _entry_level = level; + need_new_entrypoint = false; + } + remove_link_to(neighbor_id, docid, level); + } + set_link_array(docid, level, empty); + } + if (need_new_entrypoint) { + _entry_docid = 0; + _entry_level = -1; + } + search::datastore::EntryRef invalid; + _node_refs[docid].store_release(invalid); +} + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h new file mode 100644 index 00000000000..be6f507dbe1 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -0,0 +1,54 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "doc_vector_access.h" +#include "hnsw_index_base.h" +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/datastore/array_store.h> +#include <vespa/vespalib/datastore/entryref.h> +#include <vespa/vespalib/util/rcuvector.h> + +namespace search::tensor { + +class DocVectorAccess; +class RandomLevelGenerator; + +/** + * Concrete implementation of a hierarchical navigable small world graph (HNSW) + * that is used for approximate K-nearest neighbor search. + * + * See HnswIndexBase for more details. + * + * The FloatType template argument specifies the data type used in the vectors (4 byte float or 8 byte double). + */ +template <typename FloatType = float> +class HnswIndex : public HnswIndexBase { +private: + using Vector = vespalib::ConstArrayRef<FloatType>; + + inline Vector get_vector(uint32_t docid) const { + return _vectors.get_vector(docid).template typify<FloatType>(); + } + + double calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const override; + double calc_distance(const Vector& lhs, uint32_t rhs_docid) const; + /** + * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. + */ + HnswCandidate find_nearest_in_layer(const Vector& input, const HnswCandidate& entry_point, uint32_t level); + void search_layer(const Vector& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level); + +public: + HnswIndex(const DocVectorAccess& vectors, RandomLevelGenerator& level_generator, const Config& cfg); + ~HnswIndex() override; + + void add_document(uint32_t docid) override; + void remove_document(uint32_t docid) override; +}; + +template class HnswIndex<float>; +template class HnswIndex<double>; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp new file mode 100644 index 00000000000..a7b9c1dba79 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp @@ -0,0 +1,192 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_index_base.h" +#include "random_level_generator.h" +#include <vespa/vespalib/datastore/array_store.hpp> +#include <vespa/vespalib/util/rcuvector.hpp> + +namespace search::tensor { + +namespace { + +// TODO: Move this to MemoryAllocator, with name PAGE_SIZE. +constexpr size_t small_page_size = 4 * 1024; +constexpr size_t min_num_arrays_for_new_buffer = 8 * 1024; +constexpr float alloc_grow_factor = 0.2; +// TODO: Adjust these numbers to what we accept as max in config. +constexpr size_t max_level_array_size = 16; +constexpr size_t max_link_array_size = 64; + +} + +search::datastore::ArrayStoreConfig +HnswIndexBase::make_default_node_store_config() +{ + return NodeStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true); +} + +search::datastore::ArrayStoreConfig +HnswIndexBase::make_default_link_store_config() +{ + return LinkStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, + small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true); +} + +uint32_t +HnswIndexBase::max_links_for_level(uint32_t level) const +{ + return (level == 0) ? _cfg.max_links_at_level_0() : _cfg.max_links_at_hierarchic_levels(); +} + +uint32_t +HnswIndexBase::make_node_for_document(uint32_t docid) +{ + uint32_t max_level = _level_generator.max_level(); + // TODO: Add capping on num_levels + uint32_t num_levels = max_level + 1; + // Note: The level array instance lives as long as the document is present in the index. + LevelArray levels(num_levels, AtomicEntryRef()); + auto node_ref = _nodes.add(levels); + _node_refs[docid].store_release(node_ref); + return max_level; +} + +HnswIndexBase::LevelArrayRef +HnswIndexBase::get_level_array(uint32_t docid) const +{ + auto node_ref = _node_refs[docid].load_acquire(); + return _nodes.get(node_ref); +} + +HnswIndexBase::LinkArrayRef +HnswIndexBase::get_link_array(uint32_t docid, uint32_t level) const +{ + auto levels = get_level_array(docid); + assert(level < levels.size()); + return _links.get(levels[level].load_acquire()); +} + +void +HnswIndexBase::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links) +{ + auto links_ref = _links.add(links); + auto node_ref = _node_refs[docid].load_acquire(); + auto levels = _nodes.get_writable(node_ref); + levels[level].store_release(links_ref); +} + +bool +HnswIndexBase::have_closer_distance(HnswCandidate candidate, const LinkArray& result) const +{ + for (uint32_t result_docid : result) { + double dist = calc_distance(candidate.docid, result_docid); + if (dist < candidate.distance) { + return true; + } + } + return false; +} + +HnswIndexBase::LinkArray +HnswIndexBase::select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + HnswCandidateVector sorted(neighbors); + std::sort(sorted.begin(), sorted.end(), LesserDistance()); + LinkArray result; + for (size_t i = 0, m = std::min(static_cast<size_t>(max_links), sorted.size()); i < m; ++i) { + result.push_back(sorted[i].docid); + } + return result; +} + +HnswIndexBase::LinkArray +HnswIndexBase::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + LinkArray result; + bool need_filtering = neighbors.size() > max_links; + NearestPriQ nearest; + for (const auto& entry : neighbors) { + nearest.push(entry); + } + while (!nearest.empty()) { + auto candidate = nearest.top(); + nearest.pop(); + if (need_filtering && have_closer_distance(candidate, result)) { + continue; + } + result.push_back(candidate.docid); + if (result.size() == max_links) { + return result; + } + } + return result; +} + +HnswIndexBase::LinkArray +HnswIndexBase::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const +{ + if (_cfg.heuristic_select_neighbors()) { + return select_neighbors_heuristic(neighbors, max_links); + } else { + return select_neighbors_simple(neighbors, max_links); + } +} + +void +HnswIndexBase::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level) +{ + set_link_array(docid, level, neighbors); + for (uint32_t neighbor_docid : neighbors) { + auto old_links = get_link_array(neighbor_docid, level); + LinkArray new_links(old_links.begin(), old_links.end()); + new_links.push_back(docid); + set_link_array(neighbor_docid, level, new_links); + } +} + +void +HnswIndexBase::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level) +{ + LinkArray new_links; + auto old_links = get_link_array(remove_from, level); + for (uint32_t id : old_links) { + if (id != remove_id) new_links.push_back(id); + } + set_link_array(remove_from, level, new_links); +} + +HnswIndexBase::HnswIndexBase(const DocVectorAccess& vectors, RandomLevelGenerator& level_generator, const Config& cfg) + : _vectors(vectors), + _level_generator(level_generator), + _cfg(cfg), + _node_refs(), + _nodes(make_default_node_store_config()), + _links(make_default_link_store_config()), + _entry_docid(0), // Note that docid 0 is reserved and never used + _entry_level(-1) +{ +} + +HnswIndexBase::~HnswIndexBase() = default; + +HnswNode +HnswIndexBase::get_node(uint32_t docid) const +{ + auto node_ref = _node_refs[docid].load_acquire(); + if (!node_ref.valid()) { + return HnswNode(); + } + auto levels = _nodes.get(node_ref); + HnswNode::LevelArray result; + for (const auto& links_ref : levels) { + auto links = _links.get(links_ref.load_acquire()); + HnswNode::LinkArray result_links(links.begin(), links.end()); + std::sort(result_links.begin(), result_links.end()); + result.push_back(result_links); + } + return HnswNode(result); +} + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h new file mode 100644 index 00000000000..9987b61c157 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h @@ -0,0 +1,130 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "hnsw_index_utils.h" +#include "hnsw_node.h" +#include "nearest_neighbor_index.h" +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/vespalib/datastore/array_store.h> +#include <vespa/vespalib/datastore/atomic_entry_ref.h> +#include <vespa/vespalib/datastore/entryref.h> +#include <vespa/vespalib/util/rcuvector.h> + +namespace search::tensor { + +class DocVectorAccess; +class RandomLevelGenerator; + +/** + * Base class for an implementation of a hierarchical navigable small world graph (HNSW) + * that is used for approximate K-nearest neighbor search. + * + * The implementation supports 1 write thread and multiple search threads without the use of mutexes. + * This is achieved by using data stores that use generation tracking and associated memory management. + * + * The implementation is mainly based on the algorithms described in + * "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs" (Yu. A. Malkov, D. A. Yashunin), + * but some adjustments are made to support proper removes. + * + * TODO: Add details on how to handle removes. + */ +class HnswIndexBase : public NearestNeighborIndex { +public: + class Config { + private: + uint32_t _max_links_at_level_0; + uint32_t _max_links_at_hierarchic_levels; + uint32_t _neighbors_to_explore_at_construction; + bool _heuristic_select_neighbors; + + public: + Config(uint32_t max_links_at_level_0_in, + uint32_t max_links_at_hierarchic_levels_in, + uint32_t neighbors_to_explore_at_construction_in, + bool heuristic_select_neighbors_in) + : _max_links_at_level_0(max_links_at_level_0_in), + _max_links_at_hierarchic_levels(max_links_at_hierarchic_levels_in), + _neighbors_to_explore_at_construction(neighbors_to_explore_at_construction_in), + _heuristic_select_neighbors(heuristic_select_neighbors_in) + {} + uint32_t max_links_at_level_0() const { return _max_links_at_level_0; } + uint32_t max_links_at_hierarchic_levels() const { return _max_links_at_hierarchic_levels; } + uint32_t neighbors_to_explore_at_construction() const { return _neighbors_to_explore_at_construction; } + bool heuristic_select_neighbors() const { return _heuristic_select_neighbors; } + }; + +protected: + using AtomicEntryRef = search::datastore::AtomicEntryRef; + + // This uses 10 bits for buffer id -> 1024 buffers. + // As we have very short arrays we get less fragmentation with fewer and larger buffers. + using EntryRefType = search::datastore::EntryRefT<22>; + + // Provides mapping from document id -> node reference. + // The reference is used to lookup the node data in NodeStore. + using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>; + + // This stores the level arrays for all nodes. + // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. + using NodeStore = search::datastore::ArrayStore<AtomicEntryRef, EntryRefType>; + using LevelArrayRef = NodeStore::ConstArrayRef; + using LevelArray = vespalib::Array<AtomicEntryRef>; + + // This stores all link arrays. + // A link array consists of the document ids of the nodes a particular node is linked to. + using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>; + using LinkArrayRef = LinkStore::ConstArrayRef; + using LinkArray = vespalib::Array<uint32_t>; + + const DocVectorAccess& _vectors; + RandomLevelGenerator& _level_generator; + Config _cfg; + NodeRefVector _node_refs; + NodeStore _nodes; + LinkStore _links; + uint32_t _entry_docid; + int _entry_level; + + static search::datastore::ArrayStoreConfig make_default_node_store_config(); + static search::datastore::ArrayStoreConfig make_default_link_store_config(); + + uint32_t max_links_for_level(uint32_t level) const; + uint32_t make_node_for_document(uint32_t docid); + LevelArrayRef get_level_array(uint32_t docid) const; + LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; + void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); + + virtual double calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const = 0; + + /** + * Returns true if the distance between the candidate and a node in the current result + * is less than the distance between the candidate and the node we want to add to the graph. + * In this case the candidate should be discarded as we already are connected to the space + * where the candidate is located. + * Used by select_neighbors_heuristic(). + */ + bool have_closer_distance(HnswCandidate candidate, const LinkArray& curr_result) const; + LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; + LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; + LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; + void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level); + void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); + +public: + HnswIndexBase(const DocVectorAccess& vectors, RandomLevelGenerator& level_generator, const Config& cfg); + ~HnswIndexBase() override; + + // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists) + + uint32_t get_entry_docid() const { return _entry_docid; } + uint32_t get_entry_level() const { return _entry_level; } + + // Should only be used by unit tests. + HnswNode get_node(uint32_t docid) const; + + // TODO: Implement set_node() as well for use in unit tests. +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h new file mode 100644 index 00000000000..b11d3f36a7a --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -0,0 +1,48 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <queue> +#include <vector> + +namespace search::tensor { + +/** + * Represents a candidate node with its distance to another point in space. + */ +struct HnswCandidate { + uint32_t docid; + double distance; + HnswCandidate(uint32_t docid_in, double distance_in) : docid(docid_in), distance(distance_in) {} +}; + +struct GreaterDistance { + bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const { + return (rhs.distance < lhs.distance); + } +}; + +struct LesserDistance { + bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const { + return (lhs.distance < rhs.distance); + } +}; + +using HnswCandidateVector = std::vector<HnswCandidate>; + +/** + * Priority queue that keeps the candidate node that is nearest a point in space on top. + */ +using NearestPriQ = std::priority_queue<HnswCandidate, HnswCandidateVector, GreaterDistance>; + +/** + * Priority queue that keeps the candidate node that is furthest away a point in space on top. + */ +class FurthestPriQ : public std::priority_queue<HnswCandidate, HnswCandidateVector, LesserDistance> { +public: + const HnswCandidateVector& peek() const { return c; } +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h new file mode 100644 index 00000000000..8a6187ffbcc --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vector> + +namespace search::tensor { + +/** + * Represents a snapshot of a graph node with all its levels and links. + * Should only be used by unit tests. + */ +class HnswNode { +public: + using LinkArray = std::vector<uint32_t>; + using LevelArray = std::vector<LinkArray>; + +private: + LevelArray _levels; + +public: + HnswNode() : _levels() {} + HnswNode(const LinkArray& level_0) : _levels() { _levels.push_back(level_0); } + HnswNode(const LevelArray& levels_in) : _levels(levels_in) {} + bool empty() const { return _levels.empty(); } + size_t size() const { return _levels.size(); } + const LevelArray& levels() const { return _levels; } + const LinkArray& level(size_t idx) const { return _levels[idx]; } + bool operator==(const HnswNode& rhs) { + return _levels == rhs._levels; + } +}; + +} + diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h new file mode 100644 index 00000000000..2167157f6cb --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -0,0 +1,19 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::tensor { + +/** + * Interface for an index that is used for (approximate) nearest neighbor search. + */ +class NearestNeighborIndex { +public: + virtual ~NearestNeighborIndex() {} + virtual void add_document(uint32_t docid) = 0; + virtual void remove_document(uint32_t docid) = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h new file mode 100644 index 00000000000..0fcac977d9d --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h @@ -0,0 +1,16 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::tensor { + +/** + * Interface used to randomly draw the max level a new hnsw node should exist in. + */ +class RandomLevelGenerator { +public: + virtual ~RandomLevelGenerator() {} + virtual uint32_t max_level() = 0; +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp index 5ef9b6cb2d4..4c80c6f869c 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp @@ -61,4 +61,9 @@ MockAttributeManager::addAttribute(const AttributeVector::SP &attr) { addAttribute(attr->getName(), attr); } +std::shared_ptr<attribute::ReadableAttributeVector> +MockAttributeManager::readable_attribute_vector(const string& name) const { + return findAttribute(name); +} + } diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h index dbf84a40e84..a8ec686433a 100644 --- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h +++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h @@ -28,6 +28,7 @@ public: IAttributeContext::UP createContext() const override; void addAttribute(const vespalib::string &name, const AttributeVector::SP &attr); void addAttribute(const AttributeVector::SP &attr); + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; }; } diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index b161958fc43..4cad98a8e01 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -100,6 +100,10 @@ public: IAttributeContext::UP createContext() const override { return IAttributeContext::UP(new MyAttributeContext(_attr)); } + + std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override { + LOG_ABORT("should not be reached"); + } }; struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index e12ea3cf47d..886cf3e886b 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -35,7 +35,7 @@ public interface TlsContext extends AutoCloseable { "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3, Java 12 Set<String> ALLOWED_PROTOCOLS = com.yahoo.vespa.jdk8compat.Set.of("TLSv1.2"); // TODO Enable TLSv1.3 - String SSL_CONTEXT_VERSION = "TLSv1.2"; // TODO Enable TLSv1.3 + String SSL_CONTEXT_VERSION = "TLS"; // Use SSLContext implementations that supports all TLS versions /** * @return the allowed cipher suites supported by the provided context instance diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index ff1d7e1d6ec..a758ca1fbbe 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -196,6 +196,8 @@ ProcessMemoryStats::create(uint64_t sizeEpsilon) i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str()); } std::sort(samples.begin(), samples.end()); + LOG(warning, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'", + sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str()); return samples[samples.size()/2]; } diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 73873e78032..1aafe58af6c 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -7,11 +7,12 @@ vespa_add_library(storage_storageserver changedbucketownershiphandler.cpp communicationmanager.cpp communicationmanagermetrics.cpp - configurable_bucket_resolver.cpp config_logging.cpp + configurable_bucket_resolver.cpp distributornode.cpp distributornodecontext.cpp documentapiconverter.cpp + fnet_metrics_wrapper.cpp fnetlistener.cpp mergethrottler.cpp messagesink.cpp diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp new file mode 100644 index 00000000000..476ececd078 --- /dev/null +++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fnet_metrics_wrapper.h" + +namespace storage { + +FnetMetricsWrapper::FnetMetricsWrapper(metrics::MetricSet* owner) + : metrics::MetricSet("fnet", {}, "transport layer metrics", owner), + _num_connections("num-connections", {}, "total number of connection objects", this) +{ +} + +FnetMetricsWrapper::~FnetMetricsWrapper() = default; + +void +FnetMetricsWrapper::update_metrics() +{ + _num_connections.set(FNET_Connection::get_num_connections()); +} + +} diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h new file mode 100644 index 00000000000..dacb9e9f52c --- /dev/null +++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h @@ -0,0 +1,22 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/metrics/metrics.h> +#include <vespa/fnet/connection.h> + +namespace storage { + +// Simple wrapper around low-level fnet network metrics +class FnetMetricsWrapper : public metrics::MetricSet +{ +private: + metrics::LongValueMetric _num_connections; + +public: + explicit FnetMetricsWrapper(metrics::MetricSet* owner); + ~FnetMetricsWrapper() override; + void update_metrics(); +}; + +} diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index f3240f0663b..6ed5c1ab650 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -23,7 +23,8 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - tls_metrics(this) + tls_metrics(this), + fnet_metrics(this) {} StorageMetricSet::~StorageMetricSet() = default; @@ -35,6 +36,7 @@ void StorageMetricSet::updateMetrics() { // be erased from history. This will no longer be a problem once we move to a // metrics system built around absolute (rather than derived) values. tls_metrics.update_metrics_with_snapshot_delta(); + fnet_metrics.update_metrics(); } } // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index 49795c63324..a315e974c01 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -3,6 +3,7 @@ #pragma once #include "tls_statistics_metrics_wrapper.h" +#include "fnet_metrics_wrapper.h" #include <vespa/metrics/metrics.h> namespace storage { @@ -27,6 +28,7 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse_visiting; TlsStatisticsMetricsWrapper tls_metrics; + FnetMetricsWrapper fnet_metrics; StorageMetricSet(); ~StorageMetricSet() override; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java index ec8c1f3f9f3..49b10a37329 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java @@ -32,6 +32,7 @@ public class AthenzAccessToken { } public String value() { return value; } + public String valueWithBearerPrefix() { return BEARER_TOKEN_PREFIX + value; } @Override public String toString() { return "AthenzAccessToken{value='" + value + "'}"; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index eec578ddc26..c05213c8008 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -2,14 +2,17 @@ package com.yahoo.vespa.athenz.client.zts; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity; @@ -36,6 +39,7 @@ import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -147,6 +151,33 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override + public AthenzAccessToken getAccessToken(AthenzDomain domain) { + return this.getAccessTokenImpl(List.of(new AthenzResourceName(domain, "domain"))); + } + + @Override + public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) { + List<AthenzResourceName> athenzResourceNames = athenzRole.stream() + .map(AthenzRole::toResourceName) + .collect(toList()); + return this.getAccessTokenImpl(athenzResourceNames); + } + + private AthenzAccessToken getAccessTokenImpl(List<AthenzResourceName> resources) { + URI uri = ztsUrl.resolve("oauth2/token"); + RequestBuilder requestBuilder = RequestBuilder.post(uri) + .addHeader("Content-Type", "application/x-www-form-urlencoded") + .addParameter("grant_type", "client_credentials") + .addParameter("scope", resources.stream().map(AthenzResourceName::toResourceNameString).collect(Collectors.joining(" "))); + + HttpUriRequest request = requestBuilder.build(); + return execute(request, response -> { + AccessTokenResponseEntity accessTokenResponseEntity = readEntity(response, AccessTokenResponseEntity.class); + return accessTokenResponseEntity.accessToken(); + }); + } + + @Override public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) { RoleCertificateRequestEntity requestEntity = new RoleCertificateRequestEntity(csr, expiry); URI uri = ztsUrl.resolve(String.format("domain/%s/role/%s/token", role.domain().getName(), role.roleName())); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index c09ad8f48a0..baeb4271905 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.client.zts; import com.yahoo.security.Pkcs10Csr; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; @@ -78,6 +79,22 @@ public interface ZtsClient extends AutoCloseable { ZToken getRoleToken(AthenzRole athenzRole); /** + * Fetch an access token for the target domain + * + * @param domain Target domain + * @return An Athenz access token + */ + AthenzAccessToken getAccessToken(AthenzDomain domain); + + /** + * Fetch an access token for the target roles + * + * @param athenzRole List of athenz roles to get access token for + * @return An Athenz access token + */ + AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole); + + /** * Fetch role certificate for the target domain and role * * @param role Target role diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java new file mode 100644 index 00000000000..edd423b52bd --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zts.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; +import com.yahoo.vespa.athenz.api.AthenzResourceName; +import com.yahoo.vespa.athenz.api.AthenzRole; + +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AccessTokenResponseEntity { + private final AthenzAccessToken accessToken; + private final Instant expiryTime; + private final List<AthenzRole> roles; + + public AccessTokenResponseEntity( + @JsonProperty("access_token") String accessToken, + @JsonProperty("expires_in") int expiresIn, + @JsonProperty("scope") String roles) { + + this.accessToken = new AthenzAccessToken(accessToken); + // We do not know from when, so best we can do is assume now ... + this.expiryTime = Instant.now().plusSeconds(expiresIn); + this.roles = Stream.of(roles.split(" ")) + .map(AthenzResourceName::fromString) + .map(AthenzRole::fromResourceName) + .collect(Collectors.toList()); + } + + public AthenzAccessToken accessToken() { + return accessToken; + } + + public Instant expiryTime() { + return expiryTime; + } + + public List<AthenzRole> roles() { + return roles; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index bea9af458b4..5d6f0e3ce16 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -16,6 +16,7 @@ import com.yahoo.security.KeyStoreType; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.tls.MutableX509KeyManager; +import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; @@ -84,6 +85,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache; private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache; + private final LoadingCache<AthenzDomain, AthenzAccessToken> domainSpecificAccessTokenCache; + private final LoadingCache<List<AthenzRole>, AthenzAccessToken> roleSpecificAccessTokenCache; private final CsrGenerator csrGenerator; @Inject @@ -116,6 +119,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext); roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); + domainSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); + roleSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken); this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName()); this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore); registerInstance(); @@ -199,6 +204,16 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override + public String getAccessToken(String domain) { + return null; + } + + @Override + public String getAccessToken(String domain, List<String> roles) { + return null; + } + + @Override public PrivateKey getPrivateKey() { return credentials.getKeyPair().getPrivate(); } @@ -240,6 +255,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } + private AthenzAccessToken createAccessToken(AthenzDomain domain) { + try (ZtsClient client = createZtsClient()) { + return client.getAccessToken(domain); + } + } + + private AthenzAccessToken createAccessToken(List<AthenzRole> roles) { + try (ZtsClient client = createZtsClient()) { + return client.getAccessToken(roles); + } + } + private DefaultZtsClient createZtsClient() { return new DefaultZtsClient(ztsEndpoint, getIdentitySslContext()); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java index fa463a77923..32cdbf9af5c 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java @@ -69,6 +69,7 @@ public class VespaDocumentOperation extends EvalFunc<String> { private static final String SIMPLE_OBJECT_FIELDS = "simple-object-fields"; private static final String CREATE_TENSOR_FIELDS = "create-tensor-fields"; private static final String EXCLUDE_FIELDS = "exclude-fields"; + private static final String TESTSET_CONDITION = "condition"; private static final String PARTIAL_UPDATE_ASSIGN = "assign"; @@ -162,7 +163,11 @@ public class VespaDocumentOperation extends EvalFunc<String> { if (op == Operation.UPDATE && createIfNonExistent) { writeField("create", true, DataType.BOOLEAN, g, properties, schema, op, 0); } - + String testSetConditionTemplate = properties.getProperty(TESTSET_CONDITION); + if (testSetConditionTemplate != null) { + String testSetCondition = TupleTools.toString(fields, testSetConditionTemplate); + writeField(TESTSET_CONDITION, testSetCondition, DataType.CHARARRAY, g, properties, schema, op, 0); + } if (op != Operation.REMOVE) { writeField("fields", fields, DataType.MAP, g, properties, schema, op, 0); } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java index 7d0fe72fc64..3c6805019b8 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java @@ -47,6 +47,19 @@ public class VespaDocumentOperationTest { assertEquals(3, fields.get("value").get("assign").getIntValue()); } + @Test + public void requireThatUDFSupportsConditionalUpdateAssign() throws IOException { + String json = getDocumentOperationJson("docid=id:<application>:metrics::<name>-<date>", "operation=update", "condition=clicks < <value>"); + ObjectMapper m = new ObjectMapper(); + JsonNode root = m.readTree(json); + JsonNode fields = root.path("fields"); + + assertEquals("id:testapp:metrics::clicks-20160112", root.get("update").getTextValue()); + assertEquals("clicks < 3", root.get("condition").getTextValue()); + assertEquals("testapp", fields.get("application").get("assign").getTextValue()); + assertEquals("clicks", fields.get("name").get("assign").getTextValue()); + assertEquals(3, fields.get("value").get("assign").getIntValue()); + } @Test public void requireThatUDFSupportsCreateIfNonExistent() throws IOException { diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index ec8a78467ca..ed079442440 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -61,12 +61,9 @@ public class TestRunnerHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - if (path.equals("/tester/v1/log")) { + // TODO: Migrate to /tester/v1/log when /tester/v1/log2 is not in use anymore (and remove /tester/v1/log2) + if (path.equals("/tester/v1/log") || path.equals("/tester/v1/log2")) { return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after") - ? Long.parseLong(request.getProperty("after")) - : -1))); - } else if (path.equals("/tester/v1/log2")) { // TODO: Migrate to /tester/v1/log when the above is not in use anymore - return new SlimeJsonResponse(log2ToSlime(testRunner.getLog(request.hasProperty("after") ? Long.parseLong(request.getProperty("after")) : -1))); } else if (path.equals("/tester/v1/status")) { @@ -98,13 +95,6 @@ public class TestRunnerHandler extends LoggingRequestHandler { } static Slime logToSlime(Collection<LogRecord> log) { - Slime root = new Slime(); - Cursor recordArray = root.setArray(); - logArrayToSlime(recordArray, log); - return root; - } - - static Slime log2ToSlime(Collection<LogRecord> log) { Slime slime = new Slime(); Cursor root = slime.setObject(); Cursor recordArray = root.setArray("logRecords"); diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java index a40af0fd3be..77de009571b 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java @@ -1,7 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.testrunner; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -24,17 +24,8 @@ public class TestRunnerHandlerTest { Log log = new Log(); LogRecord record = log.getLogRecord(); String trace = log.getTrace(); - assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]", - new String(SlimeUtils.toJsonBytes(TestRunnerHandler.logToSlime(Collections.singletonList(record))))); - } - - @Test - public void log2Serialization() throws IOException { - Log log = new Log(); - LogRecord record = log.getLogRecord(); - String trace = log.getTrace(); assertEquals("{\"logRecords\":[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]}", - new String(SlimeUtils.toJsonBytes(TestRunnerHandler.log2ToSlime(Collections.singletonList(record))))); + new String(SlimeUtils.toJsonBytes(TestRunnerHandler.logToSlime(Collections.singletonList(record))))); } private static class Log { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java index aeddc762586..0661363477f 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -202,7 +202,7 @@ public class RestApiTest { // Get logs through some hackish fetch method. Logs is something the mocked backend write. String getLog() throws IOException { - // The mocked backend will throw a runtime exception wtih a log if delete is called three times.. + // The mocked backend will throw a runtime exception with a log if delete is called three times.. Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); HttpDelete delete = new HttpDelete(request.getUri()); doRest(delete); diff --git a/vespaclient-container-plugin/src/test/rest-api-application/services.xml b/vespaclient-container-plugin/src/test/rest-api-application/services.xml index 346740bc815..ae1b87635a9 100644 --- a/vespaclient-container-plugin/src/test/rest-api-application/services.xml +++ b/vespaclient-container-plugin/src/test/rest-api-application/services.xml @@ -2,6 +2,8 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <container version="1.0" jetty="true"> + <accesslog type="disabled"/> + <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test"> <binding>http://*/document/v1/*</binding> </handler> diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 80e93977980..4d0b3a8ff2e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.slime.*; +package com.yahoo.slime; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -9,8 +7,7 @@ import java.nio.charset.StandardCharsets; import java.util.Optional; /** - * Extra utilities/operations on slime trees that we would like to have as part of slime in the future, but - * which resides here until we have a better place to put it. + * Extra utilities/operations on slime trees. * * @author Ulf Lilleengen */ @@ -20,12 +17,7 @@ public class SlimeUtils { if (from.type() != Type.OBJECT) { throw new IllegalArgumentException("Cannot copy object: " + from); } - from.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - setObjectEntry(inspector, name, to); - } - }); + from.traverse((ObjectTraverser) (name, inspector) -> setObjectEntry(inspector, name, to)); } @@ -61,13 +53,7 @@ public class SlimeUtils { } private static void copyArray(Inspector from, final Cursor to) { - from.traverse(new ArrayTraverser() { - @Override - public void entry(int i, Inspector inspector) { - addValue(inspector, to); - } - }); - + from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to)); } private static void addValue(Inspector from, Cursor to) { diff --git a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java index ff7b640630e..8b13ee74aed 100644 --- a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; +package com.yahoo.slime; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; import com.yahoo.text.Utf8; import org.junit.Test; @@ -14,9 +12,9 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.8 */ public class SlimeUtilsTest { + @Test public void test_copying_slime_types_into_cursor() { Slime slime = new Slime(); @@ -79,4 +77,5 @@ public class SlimeUtilsTest { assertThat(slime.get().field("foo").asString(), is("foobie")); assertTrue(slime.get().field("bar").valid()); } + } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index d5fef404a43..4c289c04564 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -25,6 +25,7 @@ template <typename EntryT, typename RefT = EntryRefT<19> > class ArrayStore { public: + using ArrayRef = vespalib::ArrayRef<EntryT>; using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; using DataStoreType = DataStoreT<RefT>; using SmallArrayType = BufferType<EntryT>; @@ -82,6 +83,17 @@ public: return getLargeArray(internalRef); } } + + /** + * Returns a writeable reference to the given array. + * + * NOTE: Use with care if reader threads are accessing arrays at the same time. + * If so, replacing an element in the array should be an atomic operation. + */ + ArrayRef get_writable(EntryRef ref) { + return vespalib::unconstify(get(ref)); + } + void remove(EntryRef ref); ICompactionContext::UP compactWorst(bool compactMemory, bool compactAddressSpace); vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } diff --git a/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h new file mode 100644 index 00000000000..154c9669550 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h @@ -0,0 +1,42 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "entryref.h" +#include <atomic> + +namespace search::datastore { + +/** + * A wrapper for std::atomic of type EntryRef that supports copy and move constructors and assignment operator, + * and uses Release-Acquire ordering for store and load. + * + * Use this class when entry refs are stored in data stores or rcu vectors, + * where copy and move constructors and assignment operator are needed when resizing underlying buffers. + * In this case synchronization between the writer thread and reader threads + * is handled as part of the buffer switch. + */ +class AtomicEntryRef { +private: + std::atomic<uint32_t> _ref; + +public: + AtomicEntryRef() noexcept : _ref() {} + explicit AtomicEntryRef(EntryRef ref) noexcept : _ref(ref.ref()) {} + AtomicEntryRef(const AtomicEntryRef& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {} + AtomicEntryRef(AtomicEntryRef&& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {} + AtomicEntryRef& operator=(const AtomicEntryRef& rhs) noexcept { + uint32_t ref = rhs._ref.load(std::memory_order_relaxed); + _ref.store(ref, std::memory_order_relaxed); + return *this; + } + + void store_release(EntryRef ref) noexcept { + _ref.store(ref.ref(), std::memory_order_release); + } + EntryRef load_acquire() const noexcept { + return EntryRef(_ref.load(std::memory_order_acquire)); + } +}; + +} diff --git a/vtag.cmake b/vtag.cmake index eefbb638ab2..a7aaf433024 100644 --- a/vtag.cmake +++ b/vtag.cmake @@ -1,10 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -if(NOT EXISTS "${CMAKE_SOURCE_DIR}/dist/vtag.map") +if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map") message(FATAL_ERROR "dist/vtag.map does not exist, please run bootstrap.sh before configuring cmake" ) endif() function(get_vtag_define KEY) - file(STRINGS dist/vtag.map VALUE REGEX "${KEY}") + file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map VALUE REGEX "${KEY}") list(GET VALUE 0 LINE) separate_arguments(DATA UNIX_COMMAND "${LINE}") list(GET DATA 1 VALUE) diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java index 0981ae615df..863d2dba708 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java @@ -221,7 +221,7 @@ public class VespaZooKeeperServerImplTest { "ssl.quorum.clientAuth=NEED\n" + "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.quorum.enabledProtocols=TLSv1.2\n" + - "ssl.quorum.protocol=TLSv1.2\n"; + "ssl.quorum.protocol=TLS\n"; } private String commonTlsClientServerConfig() { @@ -229,7 +229,7 @@ public class VespaZooKeeperServerImplTest { "ssl.clientAuth=NEED\n" + "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.enabledProtocols=TLSv1.2\n" + - "ssl.protocol=TLSv1.2\n"; + "ssl.protocol=TLS\n"; } private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { |