diff options
101 files changed, 826 insertions, 287 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java index b7f4b0c7ddb..7427c882ae8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateViewTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.*; @@ -33,8 +33,6 @@ public class ClusterStateViewTest { verify(statsAggregator, never()).updateForDistributor(anyInt(), any()); } - - @Test public void testStateVersionMismatch() { when(nodeInfo.isDistributor()).thenReturn(true); @@ -100,4 +98,5 @@ public class ClusterStateViewTest { assert(indices.contains(2)); assert(indices.contains(6)); } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java index e8f13b9f741..7f81645474d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendrerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java @@ -19,7 +19,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; -public class ContentClusterHtmlRendrerTest { +public class ContentClusterHtmlRendererTest { private final VdsClusterHtmlRenderer renderer = new VdsClusterHtmlRenderer(); private final static int slobrokGeneration = 34; private final static String clusterName = "clustername"; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index 850484188a2..b8c0d2e0c54 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; @@ -26,24 +26,6 @@ public class DatabaseTest extends FleetControllerTest { private static final Logger log = Logger.getLogger(DatabaseTest.class.getName()); - // Note: different semantics than FleetControllerTest.setWantedState - private void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) { - int rpcPort = fleetController.getRpcPort(); - if (supervisor == null) { - supervisor = new Supervisor(new Transport()); - } - Target connection = supervisor.connect(new Spec("localhost", rpcPort)); - assertTrue(connection.isValid()); - - Request req = new Request("setNodeState"); - req.parameters().add(new StringValue("storage/cluster.mycluster/" + n.getType().toString() + "/" + n.getIndex())); - req.parameters().add(new StringValue(ns.serialize(true))); - connection.invokeSync(req, timeoutS); - assertEquals(req.toString(), ErrorCode.NONE, req.errorCode()); - assertTrue(req.toString(), req.checkReturnTypes("s")); - wantedStates.put(n, ns); - } - // These tests work in isolation but causes other tests to hang @Ignore @Test @@ -62,42 +44,42 @@ public class DatabaseTest extends FleetControllerTest { for (DummyVdsNode node : nodes) { wantedStates.put(node.getNode(), new NodeState(node.getType(), State.UP)); } - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); log.info("SET A WANTED STATE AND SEE THAT IT GETS PROPAGATED"); setWantedState(new Node(NodeType.STORAGE, 3), new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("Yoo"), wantedStates); waitForState("version:\\d+ distributor:10 storage:10 .3.s:m"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); log.info("SET ANOTHER WANTED STATE AND SEE THAT IT GETS PROPAGATED"); setWantedState(new Node(NodeType.DISTRIBUTOR, 2), new NodeState(NodeType.DISTRIBUTOR, State.DOWN), wantedStates); waitForState("version:\\d+ distributor:10 .2.s:d storage:10 .3.s:m"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); log.info("SET YET ANOTHER WANTED STATE AND SEE THAT IT GETS PROPAGATED"); setWantedState(new Node(NodeType.STORAGE, 7), new NodeState(NodeType.STORAGE, State.RETIRED).setDescription("We wanna replace this node"), wantedStates); waitForState("version:\\d+ distributor:10 .2.s:d storage:10 .3.s:m .7.s:r"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); log.info("CHECK THAT WANTED STATES PERSIST FLEETCONTROLLER RESTART"); stopFleetController(); startFleetController(); waitForState("version:\\d+ distributor:10 .2.s:d storage:10 .3.s:m .7.s:r"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); log.info("CLEAR WANTED STATE"); setWantedState(new Node(NodeType.STORAGE, 7), new NodeState(NodeType.STORAGE, State.UP), wantedStates); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); setWantedState(new Node(NodeType.DISTRIBUTOR, 5), new NodeState(NodeType.DISTRIBUTOR, State.DOWN), wantedStates); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); setWantedState(new Node(NodeType.DISTRIBUTOR, 2), new NodeState(NodeType.DISTRIBUTOR, State.UP), wantedStates); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); setWantedState(new Node(NodeType.STORAGE, 9), new NodeState(NodeType.STORAGE, State.DOWN), wantedStates); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); } // These tests work in isolation but causes other tests to hang @@ -119,31 +101,31 @@ public class DatabaseTest extends FleetControllerTest { wantedStates.put(node.getNode(), new NodeState(node.getType(), State.UP)); } - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); setWantedState(new Node(NodeType.STORAGE, 1), new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("Yoo"), wantedStates); waitForState("version:\\d+ distributor:10 storage:10 .1.s:m"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); - // This should not show up, as it is down + // This should not show up, as it is down setWantedState(new Node(NodeType.DISTRIBUTOR, 8), new NodeState(NodeType.DISTRIBUTOR, State.DOWN), wantedStates); waitForState("version:\\d+ distributor:10 .8.s:d storage:10 .1.s:m"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); - // This should show up, as down nodes can be turned to maintenance + // This should show up, as down nodes can be turned to maintenance setWantedState(new Node(NodeType.STORAGE, 6), new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("foobar"), wantedStates); waitForState("version:\\d+ distributor:10 .8.s:d storage:10 .1.s:m .6.s:m"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); - // This should not show up, as we cannot turn a down node retired + // This should not show up, as we cannot turn a down node retired setWantedState(new Node(NodeType.STORAGE, 7), new NodeState(NodeType.STORAGE, State.RETIRED).setDescription("foobar"), wantedStates); waitForState("version:\\d+ distributor:10 .8.s:d storage:10 .1.s:m .6.s:m .7.s:r"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); - // This should not show up, as it is down + // This should not show up, as it is down setWantedState(new Node(NodeType.STORAGE, 8), new NodeState(NodeType.STORAGE, State.DOWN).setDescription("foobar"), wantedStates); waitForState("version:\\d+ distributor:10 .8.s:d storage:10 .1.s:m .6.s:m .7.s:r .8.s:d"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); stopFleetController(); for (int i=6; i<nodes.size(); ++i) nodes.get(i).disconnect(); @@ -156,7 +138,32 @@ public class DatabaseTest extends FleetControllerTest { for (int i=6; i<nodes.size(); ++i) nodes.get(i).connect(); waitForState("version:\\d+ distributor:10 .8.s:d storage:10 .1.s:m .7.s:r .8.s:d"); - for (DummyVdsNode node : nodes) { assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); } + assertWantedStates(wantedStates); } + private void assertWantedStates(Map<Node, NodeState> wantedStates) { + for (DummyVdsNode node : nodes) { + assertEquals(node.getNode().toString(), wantedStates.get(node.getNode()), fleetController.getWantedNodeState(node.getNode())); + } + } + + // Note: different semantics than FleetControllerTest.setWantedState + private void setWantedState(Node n, NodeState ns, Map<Node, NodeState> wantedStates) { + int rpcPort = fleetController.getRpcPort(); + if (supervisor == null) { + supervisor = new Supervisor(new Transport()); + } + Target connection = supervisor.connect(new Spec("localhost", rpcPort)); + assertTrue(connection.isValid()); + + Request req = new Request("setNodeState"); + req.parameters().add(new StringValue("storage/cluster.mycluster/" + n.getType().toString() + "/" + n.getIndex())); + req.parameters().add(new StringValue(ns.serialize(true))); + connection.invokeSync(req, timeoutS); + assertEquals(req.toString(), ErrorCode.NONE, req.errorCode()); + assertTrue(req.toString(), req.checkReturnTypes("s")); + wantedStates.put(n, ns); + } + + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index 608d7d6aee3..9f5e672886e 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -42,7 +42,7 @@ public class MasterElectionTest extends FleetControllerTest { public TestRule cleanupZookeeperLogsOnSuccess = new CleanupZookeeperLogsOnSuccess(); @Rule - public Timeout globalTimeout= Timeout.seconds(120); + public Timeout globalTimeout = Timeout.seconds(120); private static int defaultZkSessionTimeoutInMillis() { return 30_000; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d713ae2e016..d7ef7fbb143 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -213,12 +213,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addParameterStoreValidationHandler(ApplicationContainerCluster cluster, DeployState deployState) { + // Always add platform bundle. Cannot be controlled by a feature flag as platform bundle cannot change. + if(deployState.isHosted()) { + cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); + } if (deployState.featureFlags().tenantIamRole()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); handler.addServerBindings(bindingPattern); - cluster.addPlatformBundle(PlatformBundles.absoluteBundlePath("jdisc-cloud-aws")); cluster.addComponent(handler); } } diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 641e2d8fde5..760d39b8f55 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index bb4ba665406..951cd8e4157 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index d05d2d1d5e0..1700cc5ff32 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT8 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -322,6 +333,7 @@ attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -380,6 +393,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -409,6 +423,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -438,6 +453,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -467,6 +483,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -496,6 +513,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index e9f3f68adc1..632ab97bbfe 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -322,6 +333,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -380,6 +393,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -409,6 +423,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -438,6 +453,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -467,6 +483,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -496,6 +513,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index 622fb9f349c..8bfc14e6795 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 4d275787bfd..188ebed6bc2 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg index bfdf90ac12c..f2a18de22bf 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index f1b20c6e454..0433c41fced 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index 9e0b5f18170..894683e6389 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 68cd917336f..16ddd67ad6d 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index a931af7af4d..900a93efe23 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 11498de54b1..93dfc9b3234 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index acbdf119d0d..35b9a1722a9 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index ecc8c2fd69d..e9471d91c45 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index a31325e67d0..e1685f55357 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 68f24871f02..0b3c39c44f6 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index ff45cf1a41e..52ad169550d 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype PREDICATE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index f9878068372..a0657f5bc27 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index ad3ff5e62f8..1517a96950c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index ebe0e83540e..e878c1f054f 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 398cdf5f8f8..71789c2987c 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index 05e19a15d96..e48d4d1bd09 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype BOOL attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent true attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -322,6 +333,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java index 8eedd156779..827241081df 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InstanceName.java @@ -11,6 +11,8 @@ import java.util.Objects; */ public class InstanceName implements Comparable<InstanceName> { + private static final InstanceName defaultInstance = new InstanceName("default"); + private final String instanceName; private InstanceName(String instanceName) { @@ -38,7 +40,7 @@ public class InstanceName implements Comparable<InstanceName> { } public static InstanceName defaultName() { - return new InstanceName("default"); + return defaultInstance; } public boolean isDefault() { diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 2a4905a2da0..b4127cd337e 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -110,7 +110,7 @@ case $1 in nohup sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS - jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000" + jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 939a0b8476a..f5b5a7cb450 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -7,6 +7,7 @@ attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SING # Deprecated/ do-not-use, will soon be GCed. attribute[].dictionary.ordering enum { ORDERED, UNORDERED } default = ORDERED attribute[].dictionary.type enum { BTREE, HASH, BTREE_AND_HASH } default = BTREE +attribute[].dictionary.match enum { CASE_SENSITIVE, CASE_INSENSITIVE } default=CASE_INSENSITIVE attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 8392f370dda..cc9d88f3fc9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -625,7 +625,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) { Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINE, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); Instant instant = Instant.now().minus(keepFileReferences); return fileReferencesOnDisk .stream() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 28211cf1d3e..52d1484897c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -123,7 +123,6 @@ public class TenantRepository { /** * Creates a new tenant repository - * */ @Inject public TenantRepository(HostRegistry hostRegistry, @@ -202,11 +201,8 @@ public class TenantRepository { curator.framework().getConnectionStateListenable().addListener(this::stateChanged); - curator.create(tenantsPath); - curator.create(locksPath); - curator.create(barriersPath); + createPaths(); createSystemTenants(configserverConfig); - curator.create(vespaPath); this.directoryCache = Optional.of(curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, zkCacheExecutor)); this.directoryCache.get().addListener(this::childEvent); @@ -260,10 +256,17 @@ public class TenantRepository { return metaData.orElse(new TenantMetaData(tenant.getName(), tenant.getCreatedTime(), tenant.getCreatedTime())); } - private static Set<TenantName> readTenantsFromZooKeeper(Curator curator) { + private static Set<TenantName> readTenantsFromZooKeeper(Curator curator) { return curator.getChildren(tenantsPath).stream().map(TenantName::from).collect(Collectors.toSet()); } + private void createPaths() { + curator.create(tenantsPath); + curator.create(locksPath); + curator.create(barriersPath); + curator.create(vespaPath); + } + private void bootstrapTenants() { // Keep track of tenants created Map<TenantName, Future<?>> futures = new HashMap<>(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index d8f818f19d9..80d01fa4d36 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; import com.yahoo.component.Version; @@ -25,9 +25,9 @@ import java.io.FilenameFilter; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; @@ -43,15 +43,21 @@ public class ZKApplicationPackageTest { private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); - private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( - Collections.singleton(new HostSpec("foo.yahoo.com", - TEST_FLAVOR.get().resources(), - TEST_FLAVOR.get().resources(), - TEST_FLAVOR.get().resources(), - ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), - Optional.of(Version.fromString("6.0.1")), Optional.empty(), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))))); + private static final AllocatedHosts ALLOCATED_HOSTS; + private static final String dockerImage = "docker.foo.com:4443/vespa/bar"; + + static { + var nodeResources = TEST_FLAVOR.orElseThrow(() -> new IllegalArgumentException("node resource found")).resources(); + ALLOCATED_HOSTS = AllocatedHosts.withHosts( + Set.of(new HostSpec("foo.yahoo.com", + nodeResources, + nodeResources, + nodeResources, + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString(dockerImage))), + Optional.of(Version.fromString("6.0.1")), Optional.empty(), + Optional.of(DockerImage.fromString(dockerImage))))); + } private ConfigCurator configCurator; @@ -92,8 +98,7 @@ public class ZKApplicationPackageTest { assertEquals(Utf8.toString(toJson(ALLOCATED_HOSTS)), Utf8.toString(toJson(readInfo))); assertEquals(TEST_FLAVOR.get().resources(), readInfo.getHosts().iterator().next().advertisedResources()); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); - // TODO: Enable when dockerImageRepo is written to zk - //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get()); + assertEquals(dockerImage, readInfo.getHosts().iterator().next().dockerImageRepo().get().asString()); assertTrue(zkApp.getDeployment().isPresent()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 8c68acc7c37..0037048fca8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -16,8 +16,11 @@ import org.jetbrains.annotations.NotNull; import java.net.URI; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; /** * This class decides which tenant goes in what bucket, and creates new buckets when required. @@ -33,6 +36,12 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { */ private final static int TENANTS_PER_BUCKET = 30; + /** + * Archive URIs are often requested because they are returned in /application/v4 API. Since they + * never change, it's safe to cache them and only update on misses + */ + private final Map<ZoneId, Map<TenantName, String>> archiveUriCache = new ConcurrentHashMap<>(); + private final ArchiveService archiveService; private final CuratorDb curatorDb; private final StringFlag bucketNameFlag; @@ -59,15 +68,16 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { } private String findOrAssignBucket(ZoneId zoneId, TenantName tenant) { - var zoneBuckets = curatorDb.readArchiveBuckets(zoneId); - return find(tenant, zoneBuckets).orElseGet(() -> assignToBucket(zoneId, tenant)); + return getBucketNameFromCache(zoneId, tenant) + .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) + .orElseGet(() -> assignToBucket(zoneId, tenant)); } private String assignToBucket(ZoneId zoneId, TenantName tenant) { try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { - Set<ArchiveBucket> zoneBuckets = new HashSet<>(curatorDb.readArchiveBuckets(zoneId)); + Set<ArchiveBucket> zoneBuckets = new HashSet<>(buckets(zoneId)); - return find(tenant, zoneBuckets) // Some other thread might have assigned it before we grabbed the lock + return findAndUpdateArchiveUriCache(zoneId, tenant, zoneBuckets) // Some other thread might have assigned it before we grabbed the lock .orElseGet(() -> { // If not, find an existing bucket with space Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() @@ -89,21 +99,36 @@ public class CuratorArchiveBucketDb implements ArchiveBucketDb { var newBucket = archiveService.createArchiveBucketFor(zoneId).withTenant(tenant); zoneBuckets.add(newBucket); curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); + updateArchiveUriCache(zoneId, zoneBuckets); return newBucket.bucketName(); }); } } + @Override + public Set<ArchiveBucket> buckets(ZoneId zoneId) { + return curatorDb.readArchiveBuckets(zoneId); + } + @NotNull - private Optional<String> find(TenantName tenant, Set<ArchiveBucket> zoneBuckets) { - return zoneBuckets.stream() + private Optional<String> findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set<ArchiveBucket> zoneBuckets) { + Optional<String> bucketName = zoneBuckets.stream() .filter(bucket -> bucket.tenants().contains(tenant)) .findAny() .map(ArchiveBucket::bucketName); + if (bucketName.isPresent()) updateArchiveUriCache(zoneId, zoneBuckets); + return bucketName; } - @Override - public Set<ArchiveBucket> buckets(ZoneId zoneId) { - return curatorDb.readArchiveBuckets(zoneId); + private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { + return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); + } + + private void updateArchiveUriCache(ZoneId zoneId, Set<ArchiveBucket> zoneBuckets) { + Map<TenantName, String> bucketNameByTenant = zoneBuckets.stream() + .flatMap(bucket -> bucket.tenants().stream() + .map(tenant -> Map.entry(tenant, bucket.bucketName()))) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + archiveUriCache.put(zoneId, bucketNameByTenant); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 16642ecbfc7..ffb5e040517 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -41,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.application.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; @@ -70,6 +69,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretSto import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.application.ActivateResult; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; @@ -1316,6 +1316,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); } + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant()) + .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); + Cursor activity = response.setObject("activity"); deployment.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", instant.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index cd9c467582c..2077278ee0c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -68,7 +68,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private HttpResponse post(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/assessment")) return new SlimeJsonResponse(doAssessment(request)); + if (path.matches("/changemanagement/v1/assessment")) return doAssessment(request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -97,31 +97,24 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Could not find any upcoming change requests with id " + changeRequestId); var changeRequest = optionalChangeRequest.get(); - var zone = affectedZone(changeRequest); - if (zone.isEmpty()) - return ErrorResponse.notFoundError("Could not find prod zone affected by change request " + changeRequestId); - - var assessment = doAssessment(changeRequest.getImpactedHosts(), zone.get()); - return new SlimeJsonResponse(assessment); + return doAssessment(changeRequest.getImpactedHosts()); } // The structure here should be // // { - // zone: string // hosts: string[] // switches: string[] // switchInSequence: boolean // } // - // Only zone and host are supported right now - private Slime doAssessment(HttpRequest request) { + // Only hosts is supported right now + private HttpResponse doAssessment(HttpRequest request) { Inspector inspector = inspectorOrThrow(request); // For now; mandatory fields - String zoneStr = getInspectorFieldOrThrow(inspector, "zone").asString(); Inspector hostArray = getInspectorFieldOrThrow(inspector, "hosts"); // The impacted hostnames @@ -130,11 +123,15 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { hostArray.traverse((ArrayTraverser) (i, host) -> hostNames.add(host.asString())); } - return doAssessment(hostNames, ZoneId.from(zoneStr)); + return doAssessment(hostNames); } - private Slime doAssessment(List<String> hostNames, ZoneId zoneId) { - ChangeManagementAssessor.Assessment assessments = assessor.assessment(hostNames, zoneId); + private HttpResponse doAssessment(List<String> hostNames) { + var zone = affectedZone(hostNames); + if (zone.isEmpty()) + return ErrorResponse.notFoundError("Could not infer prod zone from host list: " + hostNames); + + ChangeManagementAssessor.Assessment assessments = assessor.assessment(hostNames, zone.get()); Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -171,12 +168,11 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { hostObject.setLong("numberOfProblematicChildren", assessment.numberOfProblematicChildren); }); - return slime; + return new SlimeJsonResponse(slime); } - private Optional<ZoneId> affectedZone(ChangeRequest changeRequest) { - var affectedHosts = changeRequest.getImpactedHosts() - .stream() + private Optional<ZoneId> affectedZone(List<String> hosts) { + var affectedHosts = hosts.stream() .map(HostName::from) .collect(Collectors.toList()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index c69d2069650..b1b1c7ffe7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -153,6 +153,7 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void testApplicationApi() { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); // (Necessary but not provided in this API) + ((InMemoryFlagSource) tester.controller().flagSource()).withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "my-bucket"); // GET API root tester.assertResponse(request("/application/v4/", GET).userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 946593fca00..443e49a3896 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -51,6 +51,7 @@ "commit": "commit1" }, "status": "complete", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 7ba63e1664d..7181c4ee2be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -20,6 +20,7 @@ "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", "screwdriverId": "123", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 4251ba1ad95..12bd5a6efbd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -54,6 +54,7 @@ "commit": "commit1" }, "status": "complete", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 2f2e70e2cf6..cd815a2064b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; @@ -32,6 +34,7 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { tester = new ContainerTester(container, responses); addUserToHostedOperatorRole(operator); tester.serviceRegistry().configServer().nodeRepository().addNodes(ZoneId.from("prod.us-east-3"), createNodes()); + tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNode()); } @Test @@ -49,6 +52,12 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { tester.assertResponse(request, new File(filename)); } + private Node createNode() { + return new Node.Builder() + .hostname(HostName.from("host1")) + .build(); + } + private List<NodeRepositoryNode> createNodes() { List<NodeRepositoryNode> nodes = new ArrayList<>(); nodes.add(createNode("node1", "host1", "default", 0 )); diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index 6c560d9207d..6dbe1c05646 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -53,18 +53,10 @@ public class TensorFieldValue extends FieldValue { private void lazyDeserialize() { if (tensor.isEmpty() && serializedTensor.isPresent()) { - Tensor t = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(serializedTensor.get())); - if (dataType.isEmpty()) { - this.dataType = Optional.of(new TensorDataType(t.type())); - this.tensor = Optional.of(t); - } else { - if (t.type().isAssignableTo(dataType.get().getTensorType())) { - this.tensor = Optional.of(t); - } else { - throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + t.type() + - " to field of type " + dataType.get()); - } - } + var t = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(serializedTensor.get())); + Optional<Tensor> newTensor = Optional.of(t); + assignTypeFrom(newTensor); + this.tensor = newTensor; } } @@ -74,7 +66,9 @@ public class TensorFieldValue extends FieldValue { } public Optional<TensorType> getTensorType() { - lazyDeserialize(); + if (! dataType.isPresent()) { + lazyDeserialize(); + } return dataType.isPresent() ? Optional.of(dataType.get().getTensorType()) : Optional.empty(); } @@ -104,6 +98,18 @@ public class TensorFieldValue extends FieldValue { serializedTensor = Optional.empty(); } + private void assignTypeFrom(Optional<Tensor> newTensor) { + if (newTensor.isEmpty()) return; + TensorType newType = newTensor.get().type(); + if (dataType.isEmpty()) { + this.dataType = Optional.of(new TensorDataType(newType)); + } + TensorType curType = dataType.get().getTensorType(); + if (! newType.isAssignableTo(curType)) { + throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + newType + " to field of type " + curType); + } + } + @Override public void assign(Object o) { if (o == null) { @@ -111,7 +117,10 @@ public class TensorFieldValue extends FieldValue { } else if (o instanceof Tensor) { assignTensor(Optional.of((Tensor)o)); } else if (o instanceof TensorFieldValue) { - assignTensor(((TensorFieldValue)o).getTensor()); + var tfv = (TensorFieldValue)o; + assignTypeFrom(tfv.tensor); + this.serializedTensor = tfv.serializedTensor; + this.tensor = tfv.tensor; } else { throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" + o.getClass().getName() + "'."); @@ -139,17 +148,8 @@ public class TensorFieldValue extends FieldValue { * The tensor type is also set from the given tensor if it was not set before. */ public void assignTensor(Optional<Tensor> tensor) { + assignTypeFrom(tensor); this.serializedTensor = Optional.empty(); - if (tensor.isPresent()) { - if (getTensorType().isPresent() && - !tensor.get().type().isAssignableTo(getTensorType().get())) { - throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + tensor.get().type() + - " to field of type " + getTensorType().get()); - } - if (getTensorType().isEmpty()) { - this.dataType = Optional.of(new TensorDataType(tensor.get().type())); - } - } this.tensor = tensor; } diff --git a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp index a6486de6858..4a0eb00e7e7 100644 --- a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp +++ b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp @@ -5,8 +5,8 @@ #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/instruction/join_with_number_function.h> - #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/unwind_message.h> using namespace vespalib; using namespace vespalib::eval; @@ -33,11 +33,12 @@ std::ostream &operator<<(std::ostream &os, Primary primary) struct FunInfo { using LookFor = JoinWithNumberFunction; Primary primary; + bool pri_mut; bool inplace; void verify(const EvalFixture &fixture, const LookFor &fun) const { EXPECT_TRUE(fun.result_is_mutable()); EXPECT_EQUAL(fun.primary(), primary); - EXPECT_EQUAL(fun.inplace(), inplace); + EXPECT_EQUAL(fun.primary_is_mutable(), pri_mut); if (inplace) { size_t idx = (fun.primary() == Primary::LHS) ? 0 : 1; EXPECT_EQUAL(fixture.result_value().cells().data, @@ -46,17 +47,18 @@ struct FunInfo { } }; -void verify_optimized(const vespalib::string &expr, Primary primary, bool inplace) { - // fprintf(stderr, "%s\n", expr.c_str()); +void verify_optimized(const vespalib::string &expr, Primary primary, bool pri_mut) { + UNWIND_MSG("optimize %s", expr.c_str()); const CellTypeSpace stable_types(CellTypeUtils::list_stable_types(), 2); - FunInfo stable_details{primary, inplace}; + FunInfo stable_details{primary, pri_mut, pri_mut}; TEST_DO(EvalFixture::verify<FunInfo>(expr, {stable_details}, stable_types)); const CellTypeSpace unstable_types(CellTypeUtils::list_unstable_types(), 2); - TEST_DO(EvalFixture::verify<FunInfo>(expr, {}, unstable_types)); + FunInfo unstable_details{primary, pri_mut, false}; + TEST_DO(EvalFixture::verify<FunInfo>(expr, {unstable_details}, unstable_types)); } void verify_not_optimized(const vespalib::string &expr) { - // fprintf(stderr, "%s\n", expr.c_str()); + UNWIND_MSG("not: %s", expr.c_str()); CellTypeSpace all_types(CellTypeUtils::list_types(), 2); TEST_DO(EvalFixture::verify<FunInfo>(expr, {}, all_types)); } diff --git a/eval/src/vespa/eval/eval/binary_format.txt b/eval/src/vespa/eval/eval/binary_format.txt index 4d74c21005a..955f821fe00 100644 --- a/eval/src/vespa/eval/eval/binary_format.txt +++ b/eval/src/vespa/eval/eval/binary_format.txt @@ -23,7 +23,7 @@ mixed_with_cell_type[7]. (mixed tensors are tagged as both 'sparse' and 'dense') if ('with_cell_type') - 1_4_int -> cell_type (0:double, 1:float) + 1_4_int -> cell_type (0:double, 1:float, 2:bfloat16, 3:int8) if ('sparse'): 1_4_int: number of mapped dimensions -> 'n_mapped' diff --git a/eval/src/vespa/eval/instruction/join_with_number_function.cpp b/eval/src/vespa/eval/instruction/join_with_number_function.cpp index c574e3f8ad9..592076f23ce 100644 --- a/eval/src/vespa/eval/instruction/join_with_number_function.cpp +++ b/eval/src/vespa/eval/instruction/join_with_number_function.cpp @@ -14,39 +14,55 @@ namespace vespalib::eval { using Instruction = InterpretedFunction::Instruction; using State = InterpretedFunction::State; +using vespalib::eval::tensor_function::unwrap_param; +using vespalib::eval::tensor_function::wrap_param; namespace { -template <typename CT, bool inplace> -ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { - if (inplace) { +struct JoinWithNumberParam { + const ValueType res_type; + const join_fun_t function; + JoinWithNumberParam(const ValueType &r, join_fun_t f) : res_type(r), function(f) {} +}; + +template <typename ICT, typename OCT, bool inplace> +ArrayRef<OCT> make_dst_cells(ConstArrayRef<ICT> src_cells, Stash &stash) { + if constexpr (inplace) { + static_assert(std::is_same_v<ICT,OCT>); return unconstify(src_cells); } else { - return stash.create_uninitialized_array<CT>(src_cells.size()); + return stash.create_uninitialized_array<OCT>(src_cells.size()); } } -template <typename CT, typename Fun, bool inplace, bool swap> -void my_number_join_op(State &state, uint64_t param) { +template <typename ICT, typename OCT, typename Fun, bool inplace, bool swap> +void my_number_join_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinWithNumberParam>(param_in); using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; - OP my_op((join_fun_t)param); + OP my_op(param.function); const Value &tensor = state.peek(swap ? 0 : 1); - CT number = state.peek(swap ? 1 : 0).as_double(); - auto src_cells = tensor.cells().typify<CT>(); - auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); + OCT number = state.peek(swap ? 1 : 0).as_double(); + auto src_cells = tensor.cells().typify<ICT>(); + auto dst_cells = make_dst_cells<ICT, OCT, inplace>(src_cells, state.stash); apply_op2_vec_num(dst_cells.begin(), src_cells.begin(), number, dst_cells.size(), my_op); if (inplace) { state.pop_pop_push(tensor); } else { - state.pop_pop_push(state.stash.create<ValueView>(tensor.type(), tensor.index(), TypedCells(dst_cells))); + state.pop_pop_push(state.stash.create<ValueView>(param.res_type, tensor.index(), TypedCells(dst_cells))); } } struct SelectJoinWithNumberOp { - template<typename CT, typename Fun, - typename InputIsMutable, typename NumberWasLeft> + template<typename CM, typename Fun, + typename PrimaryMutable, typename NumberWasLeft> static auto invoke() { - return my_number_join_op<CT, Fun, InputIsMutable::value, NumberWasLeft::value>; + constexpr CellMeta icm = CM::value; + constexpr CellMeta num(CellType::DOUBLE, true); + constexpr CellMeta ocm = CellMeta::join(icm, num); + using ICT = CellValueType<icm.cell_type>; + using OCT = CellValueType<ocm.cell_type>; + constexpr bool inplace = (PrimaryMutable::value && std::is_same_v<ICT,OCT>); + return my_number_join_op<ICT, OCT, Fun, inplace, NumberWasLeft::value>; } }; @@ -62,7 +78,7 @@ JoinWithNumberFunction::JoinWithNumberFunction(const Join &original, bool tensor JoinWithNumberFunction::~JoinWithNumberFunction() = default; bool -JoinWithNumberFunction::inplace() const { +JoinWithNumberFunction::primary_is_mutable() const { if (_primary == Primary::LHS) { return lhs().result_is_mutable(); } else { @@ -70,16 +86,19 @@ JoinWithNumberFunction::inplace() const { } } -using MyTypify = TypifyValue<TypifyCellType,vespalib::TypifyBool,operation::TypifyOp2>; +using MyTypify = TypifyValue<TypifyCellMeta,vespalib::TypifyBool,operation::TypifyOp2>; InterpretedFunction::Instruction -JoinWithNumberFunction::compile_self(const ValueBuilderFactory &, Stash &) const +JoinWithNumberFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const { - auto op = typify_invoke<4,MyTypify,SelectJoinWithNumberOp>(result_type().cell_type(), + const auto ¶m = stash.create<JoinWithNumberParam>(result_type(), _function); + auto input_type = (_primary == Primary::LHS) ? lhs().result_type() : rhs().result_type(); + assert(result_type() == ValueType::join(input_type, ValueType::double_type())); + auto op = typify_invoke<4,MyTypify,SelectJoinWithNumberOp>(input_type.cell_meta(), _function, - inplace(), + primary_is_mutable(), (_primary == Primary::RHS)); - return Instruction(op, (uint64_t)(_function)); + return Instruction(op, wrap_param<JoinWithNumberParam>(param)); } void @@ -87,7 +106,7 @@ JoinWithNumberFunction::visit_self(vespalib::ObjectVisitor &visitor) const { Super::visit_self(visitor); visitor.visitBool("tensor_was_right", (_primary == Primary::RHS)); - visitor.visitBool("is_inplace", inplace()); + visitor.visitBool("primary_is_mutable", primary_is_mutable()); } const TensorFunction & @@ -95,17 +114,12 @@ JoinWithNumberFunction::optimize(const TensorFunction &expr, Stash &stash) { if (! expr.result_type().is_double()) { if (const auto *join = as<Join>(expr)) { - const ValueType &result_type = join->result_type(); const TensorFunction &lhs = join->lhs(); const TensorFunction &rhs = join->rhs(); - if (lhs.result_type().is_double() && - (result_type == rhs.result_type())) - { + if (lhs.result_type().is_double()) { return stash.create<JoinWithNumberFunction>(*join, true); } - if (rhs.result_type().is_double() && - (result_type == lhs.result_type())) - { + if (rhs.result_type().is_double()) { return stash.create<JoinWithNumberFunction>(*join, false); } } diff --git a/eval/src/vespa/eval/instruction/join_with_number_function.h b/eval/src/vespa/eval/instruction/join_with_number_function.h index 546ff75b175..9d29ad5eb5d 100644 --- a/eval/src/vespa/eval/instruction/join_with_number_function.h +++ b/eval/src/vespa/eval/instruction/join_with_number_function.h @@ -23,7 +23,7 @@ public: JoinWithNumberFunction(const tensor_function::Join &original_join, bool number_on_left); ~JoinWithNumberFunction(); Primary primary() const { return _primary; } - bool inplace() const; + bool primary_is_mutable() const; bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8a088860e97..460741a6b8e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -153,7 +153,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_PENDING_MOVE_OPS = defineIntFlag( - "max-pending-move-ops", 10, + "max-pending-move-ops", 100, List.of("baldersheim"), "2021-02-15", "2021-05-01", "Max number of move operations inflight", "Takes effect at redeployment", @@ -167,14 +167,14 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT = defineFeatureFlag( - "use-bucket-executor-for-lid-space-compact", false, + "use-bucket-executor-for-lid-space-compact", true, List.of("baldersheim"), "2021-01-24", "2021-05-01", "Wheter to use content-level bucket executor or legacy frozen buckets", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_BUCKET_MOVE = defineFeatureFlag( - "use-bucket-executor-for-bucket-move", false, + "use-bucket-executor-for-bucket-move", true, List.of("baldersheim"), "2021-02-15", "2021-05-01", "Wheter to use content-level bucket executor or legacy frozen buckets", "Takes effect on next internal redeployment", diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index fe4b5ebbb64..6af8c076246 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -Djava.io.tmpdir=${VESPA_HOME}/tmp" +addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index bef4864fb6d..619c62fae43 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -3,14 +3,15 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.util.Timeout; + import java.util.logging.Level; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.time.Clock; import java.time.Duration; @@ -103,11 +104,11 @@ public class ApplicationMetricsRetriever extends AbstractComponent { } private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create(PoolingHttpClientConnectionManager::new) + return VespaHttpClientBuilder.create() .setUserAgent("application-metrics-retriever") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(HTTP_CONNECT_TIMEOUT) - .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .setConnectTimeout(Timeout.ofMilliseconds(HTTP_CONNECT_TIMEOUT)) + .setResponseTimeout(Timeout.ofMilliseconds(HTTP_SOCKET_TIMEOUT)) .build()) .build(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index 78bc3e96322..2e17443e821 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -6,9 +6,9 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil; import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import java.io.IOException; import java.time.Clock; @@ -70,7 +70,7 @@ public class NodeMetricsClient { log.log(FINE, () -> "Retrieving metrics from host " + metricsUri); try { - String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicResponseHandler()); + String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicHttpClientResponseHandler()); var metricsBuilders = GenericJsonUtil.toMetricsPackets(metricsJson); var metrics = processAndBuild(metricsBuilders, new ServiceIdDimensionProcessor(), diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 05a2b85af68..2348f65bc9f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -1,13 +1,18 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.logging.Level; + import com.yahoo.yolean.Exceptions; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URI; @@ -28,8 +33,7 @@ public abstract class HttpMetricFetcher { private final static int SOCKET_TIMEOUT = 60000; private final URI url; protected final VespaService service; - private static final CloseableHttpClient httpClient = createHttpClient(); - + private static final CloseableHttpAsyncClient httpClient = createHttpClient(); /** * @param service the service to fetch metrics from @@ -43,9 +47,14 @@ public abstract class HttpMetricFetcher { log.log(Level.FINE, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); } - String getJson() throws IOException { + byte [] getJson() throws IOException { log.log(Level.FINE, "Connecting to url " + url + " for service '" + service + "'"); - return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); + Future<SimpleHttpResponse> response = httpClient.execute(new SimpleHttpRequest("GET", url), null); + try { + return response.get().getBodyBytes(); + } catch (InterruptedException | ExecutionException e) { + throw new IOException("Failed fetching '" + url + "': " + e); + } } public String toString() { @@ -57,7 +66,7 @@ public abstract class HttpMetricFetcher { Exceptions.toMessageString(e); } - void handleException(Exception e, String data, int timesFetched) { + void handleException(Exception e, Object data, int timesFetched) { logMessage("Unable to parse json '" + data + "' for service '" + service + "': " + Exceptions.toMessageString(e), timesFetched); } @@ -78,14 +87,16 @@ public abstract class HttpMetricFetcher { } } - private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create() + private static CloseableHttpAsyncClient createHttpClient() { + CloseableHttpAsyncClient client = VespaAsyncHttpClientBuilder.create() .setUserAgent("metrics-proxy-http-client") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(CONNECTION_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT) + .setConnectTimeout(Timeout.ofMilliseconds(CONNECTION_TIMEOUT)) + .setResponseTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) .build()) .build(); + client.start(); + return client; } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index f9445e5b26a..e43aab8b26f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -27,8 +27,12 @@ public class MetricsParser { private static final ObjectMapper jsonMapper = new ObjectMapper(); static Metrics parse(String data) throws IOException { - JsonParser parser = jsonMapper.createParser(data); - + return parse(jsonMapper.createParser(data)); + } + static Metrics parse(byte [] data) throws IOException { + return parse(jsonMapper.createParser(data)); + } + private static Metrics parse(JsonParser parser) throws IOException { if (parser.nextToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of object, got " + parser.currentToken()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 827f513a418..f078081c430 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -29,7 +29,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public HealthMetric getHealth(int fetchCount) { - String data = "{}"; + byte [] data = {'{', '}'}; try { data = getJson(); } catch (IOException e) { @@ -41,7 +41,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { /** * Connect to remote service over http and fetch metrics */ - private HealthMetric createHealthMetrics(String data, int fetchCount) { + private HealthMetric createHealthMetrics(byte [] data, int fetchCount) { HealthMetric healthMetric = HealthMetric.getDown("Failed fetching status page for service"); try { healthMetric = parse(data); @@ -51,8 +51,8 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { return healthMetric; } - private HealthMetric parse(String data) { - if (data == null || data.isEmpty()) { + private HealthMetric parse(byte [] data) { + if ((data == null) || (data.length == 0)) { return HealthMetric.getUnknown("Empty response from status page"); } try { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 72f77926099..787f0e33157 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -39,4 +39,14 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { return remoteMetrics; } + Metrics createMetrics(byte [] data, int fetchCount) { + Metrics remoteMetrics = new Metrics(); + try { + remoteMetrics = MetricsParser.parse(data); + } catch (Exception e) { + handleException(e, data, fetchCount); + } + + return remoteMetrics; + } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java index eba32941620..ab84a4edcde 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java @@ -5,8 +5,9 @@ import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.yahoo.test.ManualClock; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 1f4bd55525d..80dfafb5116 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -199,9 +199,20 @@ public final class Node implements Nodelike { * If both given wantToRetire and wantToDeprovision are equal to the current values, the method is no-op. */ public Node withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, Agent agent, Instant at) { + return withWantToRetire(wantToRetire, wantToDeprovision, false, agent, at); + } + + /** + * Returns a copy of this node with wantToRetire, wantToDeprovision and wantToRebuild set to the given values + * and updated history. + * + * If all given values are equal to the current ones, the method is no-op. + */ + public Node withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, boolean wantToRebuild, Agent agent, Instant at) { if (wantToRetire == status.wantToRetire() && - wantToDeprovision == status.wantToDeprovision()) return this; - Node node = this.with(status.withWantToRetire(wantToRetire, wantToDeprovision)); + wantToDeprovision == status.wantToDeprovision() && + wantToRebuild == status.wantToRebuild()) return this; + Node node = this.with(status.withWantToRetire(wantToRetire, wantToDeprovision, wantToRebuild)); if (wantToRetire) node = node.with(history.with(new History.Event(History.Event.Type.wantToRetire, agent, at))); return node; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 197193fafa9..24f94b9d63b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -129,6 +129,11 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { return matching(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false)); } + /** Returns the subset of nodes allocated to a tester instance */ + public NodeList tester() { + return matching(node -> node.allocation().isPresent() && node.allocation().get().owner().instance().isTester()); + } + /** Returns the subset of nodes matching the given node type(s) */ public NodeList nodeType(NodeType first, NodeType... rest) { if (rest.length == 0) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index ca18028ad5a..05a98c5b4da 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -83,7 +83,7 @@ public class NodeRepoStats { Map<String, NodeMetricSnapshot> snapshotsByHost = byHost(allNodeTimeseries); for (var applicationNodes : allNodes.state(Node.State.active) .nodeType(NodeType.tenant) - .matching(node -> ! node.allocation().get().owner().instance().isTester()) + .not().tester() .groupingBy(node -> node.allocation().get().owner()).entrySet()) { NodeResources totalResources = NodeResources.zero(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index fb31d51abfc..e2316f96c16 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -97,7 +97,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { List<String> unparkedChildren = !candidate.type().isHost() ? List.of() : nodeRepository.nodes().list() .childrenOf(candidate) - .matching(node -> node.state() != Node.State.parked) + .not().state(Node.State.parked) .mapToList(Node::hostname); if (unparkedChildren.isEmpty()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 2950de285b9..fe2fb5229f9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -78,7 +78,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { private void updateActiveNodeDownState() { NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName().forEach((hostname, serviceInstances) -> { - Optional<Node> node = activeNodes.matching(n -> n.hostname().equals(hostname.toString())).first(); + Optional<Node> node = activeNodes.node(hostname.toString()); if (node.isEmpty()) return; // Already correct record, nothing to do diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index 025c8be449c..0a1f6961f9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -44,7 +44,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { return nodeRepository().nodes() .list(Node.State.active) .nodeType(NodeType.tenant) - .matching(node -> ! node.allocation().get().owner().instance().isTester()) + .not().tester() .groupingBy(node -> node.allocation().get().owner()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index a04e305242f..1b7c629416a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -220,6 +220,18 @@ public class IP { ipv6 } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IpAddresses that = (IpAddresses) o; + return ipAddresses.equals(that.ipAddresses) && protocol == that.protocol; + } + + @Override + public int hashCode() { + return Objects.hash(ipAddresses, protocol); + } } /** @@ -346,13 +358,13 @@ public class IP { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Pool that = (Pool) o; - return Objects.equals(ipAddresses, that.ipAddresses); + Pool pool = (Pool) o; + return ipAddresses.equals(pool.ipAddresses) && addresses.equals(pool.addresses); } @Override public int hashCode() { - return Objects.hash(ipAddresses); + return Objects.hash(ipAddresses, addresses); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index f12699e0d81..cd0928ef320 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; @@ -198,6 +197,22 @@ public class Nodes { return setReady(List.of(nodeToReady), agent, reason).get(0); } + /** Restore a node that has been rebuilt */ + public Node restore(String hostname, Agent agent, String reason) { + // A deprovisioned host has no children so this doesn't need to to be recursive + try (NodeMutex lock = lockAndGetRequired(hostname)) { + Node existing = lock.node(); + if (existing.state() != Node.State.deprovisioned) illegal("Can not move node " + hostname + " to " + + Node.State.provisioned + ". It is not in " + + Node.State.deprovisioned); + if (!existing.status().wantToRebuild()) illegal("Can not move node " + hostname + " to " + + Node.State.provisioned + + ". Rebuild has not been requested"); + Node nodeWithResetFields = existing.withWantToRetire(false, false, false, agent, clock.instant()); + return db.writeTo(Node.State.provisioned, nodeWithResetFields, agent, Optional.of(reason)); + } + } + /** Reserve nodes. This method does <b>not</b> lock the node repository */ public List<Node> reserve(List<Node> nodes) { return db.writeTo(Node.State.reserved, nodes, Agent.application, Optional.empty()); @@ -283,20 +298,12 @@ public class Nodes { } public Node deallocate(Node node, Agent agent, String reason, NestedTransaction transaction) { - if (node.state() != Node.State.parked && agent != Agent.operator - && (node.status().wantToDeprovision() || retiredByOperator(node))) + if (parkOnDeallocationOf(node, agent)) return park(node.hostname(), false, agent, reason, transaction); else return db.writeTo(Node.State.dirty, List.of(node), agent, Optional.of(reason), transaction).get(0); } - private static boolean retiredByOperator(Node node) { - return node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire) - .map(History.Event::agent) - .map(agent -> agent == Agent.operator) - .orElse(false); - } - /** * Fails this node and returns it in its new state. * @@ -360,9 +367,7 @@ public class Nodes { * Moves a host to breakfixed state, removing any children. */ public List<Node> breakfixRecursively(String hostname, Agent agent, String reason) { - Node node = node(hostname).orElseThrow(() -> - new NoSuchNodeException("Could not breakfix " + hostname + ": Node not found")); - + Node node = node(hostname).orElseThrow(() -> new NoSuchNodeException("Could not breakfix " + hostname + ": Node not found")); try (Mutex lock = lockUnallocated()) { requireBreakfixable(node); List<Node> removed = removeChildren(node, false); @@ -389,9 +394,7 @@ public class Nodes { private Node move(String hostname, boolean keepAllocation, Node.State toState, Agent agent, Optional<String> reason, NestedTransaction transaction) { - Node node = node(hostname).orElseThrow(() -> - new NoSuchNodeException("Could not move " + hostname + " to " + toState + ": Node not found")); - + Node node = node(hostname).orElseThrow(() -> new NoSuchNodeException("Could not move " + hostname + " to " + toState + ": Node not found")); if (!keepAllocation && node.allocation().isPresent()) { node = node.withoutAllocation(); } @@ -464,7 +467,9 @@ public class Nodes { if (zone.getCloud().dynamicProvisioning() || node.type() != NodeType.host) db.removeNodes(List.of(node)); else { - node = node.with(IP.Config.EMPTY); + if (!node.status().wantToRebuild()) { // Keep IP addresses if we're rebuilding + node = node.with(IP.Config.EMPTY); + } move(node, Node.State.deprovisioned, Agent.system, Optional.empty()); } removed.add(node); @@ -582,19 +587,31 @@ public class Nodes { } /** Retire and deprovision given host and all of its children */ - public List<Node> deprovision(Node host, Agent agent, Instant instant) { - if (!host.type().isHost()) throw new IllegalArgumentException("Cannot deprovision non-host " + host); - Optional<NodeMutex> nodeMutex = lockAndGet(host); + public List<Node> deprovision(String hostname, Agent agent, Instant instant) { + return decomission(hostname, DecommisionOperation.deprovision, agent, instant); + } + + /** Retire and rebuild given host and all of its children */ + public List<Node> rebuild(String hostname, Agent agent, Instant instant) { + return decomission(hostname, DecommisionOperation.rebuild, agent, instant); + } + + private List<Node> decomission(String hostname, DecommisionOperation op, Agent agent, Instant instant) { + Optional<NodeMutex> nodeMutex = lockAndGet(hostname); if (nodeMutex.isEmpty()) return List.of(); + Node host = nodeMutex.get().node(); + if (!host.type().isHost()) throw new IllegalArgumentException("Cannot " + op + " non-host " + host); List<Node> result; + boolean wantToDeprovision = op == DecommisionOperation.deprovision; + boolean wantToRebuild = op == DecommisionOperation.rebuild; try (NodeMutex lock = nodeMutex.get(); Mutex allocationLock = lockUnallocated()) { // This takes allocationLock to prevent any further allocation of nodes on this host host = lock.node(); NodeList children = list(allocationLock).childrenOf(host); result = performOn(NodeListFilter.from(children.asList()), - (node, nodeLock) -> write(node.withWantToRetire(true, true, agent, instant), + (node, nodeLock) -> write(node.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), nodeLock)); - result.add(write(host.withWantToRetire(true, true, agent, instant), lock)); + result.add(write(host.withWantToRetire(true, wantToDeprovision, wantToRebuild, agent, instant), lock)); } return result; } @@ -747,4 +764,24 @@ public class Nodes { throw new IllegalArgumentException(message); } + /** Returns whether node should be parked when deallocated by given agent */ + private static boolean parkOnDeallocationOf(Node node, Agent agent) { + if (node.state() == Node.State.parked) return false; + if (agent == Agent.operator) return false; + boolean retirementRequestedByOperator = node.status().wantToRetire() && + node.history().event(History.Event.Type.wantToRetire) + .map(History.Event::agent) + .map(a -> a == Agent.operator) + .orElse(false); + return node.status().wantToDeprovision() || + node.status().wantToRebuild() || + retirementRequestedByOperator; + } + + /** The different ways a host can be decomissioned */ + private enum DecommisionOperation { + deprovision, + rebuild, + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 43fc07ea2c3..8964977091c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -21,6 +21,7 @@ public class Status { private final int failCount; private final boolean wantToRetire; private final boolean wantToDeprovision; + private final boolean wantToRebuild; private final boolean preferToRetire; private final OsVersion osVersion; private final Optional<Instant> firmwareVerifiedAt; @@ -31,6 +32,7 @@ public class Status { int failCount, boolean wantToRetire, boolean wantToDeprovision, + boolean wantToRebuild, boolean preferToRetire, OsVersion osVersion, Optional<Instant> firmwareVerifiedAt) { @@ -38,46 +40,50 @@ public class Status { this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null").filter(v -> !Version.emptyVersion.equals(v)); this.containerImage = Objects.requireNonNull(containerImage, "Container image must be non-null").filter(d -> !DockerImage.EMPTY.equals(d)); this.failCount = failCount; - if (wantToDeprovision && !wantToRetire) { - throw new IllegalArgumentException("Node cannot be marked wantToDeprovision unless it's also marked wantToRetire"); + if (wantToDeprovision && wantToRebuild) { + throw new IllegalArgumentException("Node cannot be marked both wantToDeprovision and wantToRebuild"); + } + if ((wantToDeprovision || wantToRebuild) && !wantToRetire) { + throw new IllegalArgumentException("Node cannot be marked wantToDeprovision or wantToRebuild unless it's also marked wantToRetire"); } this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; + this.wantToRebuild = wantToRebuild; this.preferToRetire = preferToRetire; this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); this.firmwareVerifiedAt = Objects.requireNonNull(firmwareVerifiedAt, "Firmware check instant must be non-null"); } /** Returns a copy of this with the reboot generation changed */ - public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the reboot generation of this node */ public Generation reboot() { return reboot; } /** Returns a copy of this with the vespa version changed */ - public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the Vespa version installed on the node, if known */ public Optional<Version> vespaVersion() { return vespaVersion; } /** Returns a copy of this with the container image changed */ - public Status withContainerImage(DockerImage containerImage) { return new Status(reboot, vespaVersion, Optional.of(containerImage), failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withContainerImage(DockerImage containerImage) { return new Status(reboot, vespaVersion, Optional.of(containerImage), failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the container image the node is running, if any */ public Optional<DockerImage> containerImage() { return containerImage; } - public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount + 1, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount + 1, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } - public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount - 1, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount - 1, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } - public Status withFailCount(int value) { return new Status(reboot, vespaVersion, containerImage, value, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } + public Status withFailCount(int value) { return new Status(reboot, vespaVersion, containerImage, value, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - /** Returns a copy of this with the want to retire/deprovision flags changed */ - public Status withWantToRetire(boolean wantToRetire, boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); + /** Returns a copy of this with the want to retire/deprovision/rebuild flags changed */ + public Status withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, boolean wantToRebuild) { + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** @@ -88,13 +94,16 @@ public class Status { return wantToRetire; } - /** - * Returns whether this node should be de-provisioned when possible. - */ + /** Returns whether this node should be de-provisioned when possible. */ public boolean wantToDeprovision() { return wantToDeprovision; } + /** Returns whether this node should be rebuilt when possible. */ + public boolean wantToRebuild() { + return wantToRebuild; + } + /** * Returns whether this node is requested to retire. Unlike {@link Status#wantToRetire()}, this is a soft * request to retire, which will not allow any replacement to increase node skew in the cluster. @@ -105,12 +114,12 @@ public class Status { /** Returns a copy of this with prefer-to-retire set to given value */ public Status withPreferToRetire(boolean preferToRetire) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns a copy of this with the OS version set to given version */ public Status withOsVersion(OsVersion version) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, version, firmwareVerifiedAt); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, version, firmwareVerifiedAt); } /** Returns the OS version of this node */ @@ -120,7 +129,7 @@ public class Status { /** Returns a copy of this with the firmwareVerifiedAt set to the given instant. */ public Status withFirmwareVerifiedAt(Instant instant) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, Optional.of(instant)); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, wantToRebuild, preferToRetire, osVersion, Optional.of(instant)); } /** Returns the last time this node had firmware that was verified to be up to date. */ @@ -131,7 +140,7 @@ public class Status { /** Returns the initial status of a newly provisioned node */ public static Status initial() { return new Status(Generation.initial(), Optional.empty(), Optional.empty(), 0, false, - false, false, OsVersion.EMPTY, Optional.empty()); + false, false, false, OsVersion.EMPTY, Optional.empty()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index 930db265066..f378a4249f4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -63,7 +63,7 @@ public class RetiringOsUpgrader implements OsUpgrader { LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " + host.status().osVersion().current().map(Version::toFullString).orElse("<unset>") + ", want " + target); - nodeRepository.nodes().deprovision(host, Agent.RetiringUpgrader, now); + nodeRepository.nodes().deprovision(host.hostname(), Agent.RetiringUpgrader, now); nodeRepository.nodes().upgradeOs(NodeListFilter.from(host), Optional.of(target)); nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, host.type())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 9d36be67431..5c006f6d6a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -82,6 +82,7 @@ public class NodeSerializer { private static final String nodeTypeKey = "type"; private static final String wantToRetireKey = "wantToRetire"; private static final String wantToDeprovisionKey = "wantToDeprovision"; + private static final String wantToRebuildKey = "wantToRebuild"; private static final String preferToRetireKey = "preferToRetire"; private static final String osVersionKey = "osVersion"; private static final String wantedOsVersionKey = "wantedOsVersion"; @@ -164,6 +165,7 @@ public class NodeSerializer { object.setBool(wantToRetireKey, node.status().wantToRetire()); object.setBool(preferToRetireKey, node.status().preferToRetire()); object.setBool(wantToDeprovisionKey, node.status().wantToDeprovision()); + object.setBool(wantToRebuildKey, node.status().wantToRebuild()); node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey))); toSlime(node.history(), object.setArray(historyKey)); object.setString(nodeTypeKey, toString(node.type())); @@ -271,6 +273,7 @@ public class NodeSerializer { (int) object.field(failCountKey).asLong(), object.field(wantToRetireKey).asBool(), object.field(wantToDeprovisionKey).asBool(), + object.field(wantToRebuildKey).asBool(), object.field(preferToRetireKey).asBool(), new OsVersion(versionFromSlime(object.field(osVersionKey)), versionFromSlime(object.field(wantedOsVersionKey))), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index e8fa4f04eed..1bea7056790 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -54,6 +54,7 @@ public class NodePatcher implements AutoCloseable { private static final String WANT_TO_RETIRE = "wantToRetire"; private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; + private static final String WANT_TO_REBUILD = "wantToRebuild"; private static final Set<String> RECURSIVE_FIELDS = Set.of(WANT_TO_RETIRE); private final NodeRepository nodeRepository; @@ -141,11 +142,17 @@ public class NodePatcher implements AutoCloseable { return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), memoizedNodes.get()); case "additionalHostnames" : return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withAddresses(asAddressList(value)))), memoizedNodes.get()); - case WANT_TO_RETIRE : - case WANT_TO_DEPROVISION : + case WANT_TO_RETIRE: + case WANT_TO_DEPROVISION: + case WANT_TO_REBUILD: boolean wantToRetire = asOptionalBoolean(root.field(WANT_TO_RETIRE)).orElse(node.status().wantToRetire()); boolean wantToDeprovision = asOptionalBoolean(root.field(WANT_TO_DEPROVISION)).orElse(node.status().wantToDeprovision()); - return node.withWantToRetire(wantToRetire, wantToDeprovision && !applyingAsChild, Agent.operator, clock.instant()); + boolean wantToRebuild = asOptionalBoolean(root.field(WANT_TO_REBUILD)).orElse(node.status().wantToRebuild()); + return node.withWantToRetire(wantToRetire, + wantToDeprovision && !applyingAsChild, + wantToRebuild && !applyingAsChild, + Agent.operator, + clock.instant()); case "reports" : return nodeWithPatchedReports(node, value); case "openStackId" : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index c9483a99a69..0875bf3815d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -135,27 +135,31 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { // Check paths to disallow illegal state changes if (path.matches("/nodes/v2/state/ready/{hostname}")) { nodeRepository.nodes().markNodeAvailableForNewAllocation(path.get("hostname"), Agent.operator, "Readied through the nodes/v2 API"); - return new MessageResponse("Moved " + path.get("hostname") + " to ready"); + return new MessageResponse("Moved " + path.get("hostname") + " to " + Node.State.ready); } else if (path.matches("/nodes/v2/state/failed/{hostname}")) { List<Node> failedNodes = nodeRepository.nodes().failRecursively(path.get("hostname"), Agent.operator, "Failed through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed"); + return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to " + Node.State.failed); } else if (path.matches("/nodes/v2/state/parked/{hostname}")) { List<Node> parkedNodes = nodeRepository.nodes().parkRecursively(path.get("hostname"), Agent.operator, "Parked through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked"); + return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to " + Node.State.parked); } else if (path.matches("/nodes/v2/state/dirty/{hostname}")) { List<Node> dirtiedNodes = nodeRepository.nodes().deallocateRecursively(path.get("hostname"), Agent.operator, "Dirtied through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty"); + return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to " + Node.State.dirty); } else if (path.matches("/nodes/v2/state/active/{hostname}")) { nodeRepository.nodes().reactivate(path.get("hostname"), Agent.operator, "Reactivated through nodes/v2 API"); - return new MessageResponse("Moved " + path.get("hostname") + " to active"); + return new MessageResponse("Moved " + path.get("hostname") + " to " + Node.State.active); } else if (path.matches("/nodes/v2/state/breakfixed/{hostname}")) { List<Node> breakfixedNodes = nodeRepository.nodes().breakfixRecursively(path.get("hostname"), Agent.operator, "Breakfixed through the nodes/v2 API"); - return new MessageResponse("Breakfixed " + hostnamesAsString(breakfixedNodes)); + return new MessageResponse("Moved " + hostnamesAsString(breakfixedNodes) + " to " + Node.State.breakfixed); + } + else if (path.matches("/nodes/v2/state/provisioned/{hostname}")) { + Node restoredNode = nodeRepository.nodes().restore(path.get("hostname"), Agent.operator, "Restored through the nodes/v2 API"); + return new MessageResponse("Moved " + hostnamesAsString(List.of(restoredNode)) + " to " + Node.State.provisioned); } throw new NotFoundException("Cannot put to path '" + path + "'"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 6d27acf77d1..79a6101750e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -119,7 +119,7 @@ public class MockNodeRepository extends NodeRepository { nodes.add(node10); Node node55 = Node.create("node55", ipConfig(55), "host55.yahoo.com", resources(2, 8, 50, 1, fast, local), NodeType.tenant) - .status(Status.initial().withWantToRetire(true, true)).build(); + .status(Status.initial().withWantToRetire(true, true, false)).build(); nodes.add(node55); /* Setup docker hosts (two of these will be reserved for spares */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index d18d2bf101d..c0699ebf835 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -17,6 +17,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -204,6 +205,42 @@ public class NodeRepositoryTest { } @Test + public void restore_rebuilt_host() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + assertEquals(0, tester.nodeRepository().nodes().list().size()); + + String host1 = "host1"; + String host2 = "host2"; + tester.addHost("id1", host1, "default", NodeType.host); + tester.addHost("id2", host2, "default", NodeType.host); + assertEquals(2, tester.nodeRepository().nodes().list().size()); + + // One host is requested to rebuild, two hosts are parked + tester.nodeRepository().nodes().rebuild(host2, Agent.system, tester.clock().instant()); + tester.nodeRepository().nodes().park(host1, false, Agent.system, getClass().getSimpleName()); + tester.nodeRepository().nodes().park(host2, false, Agent.system, getClass().getSimpleName()); + IP.Config ipConfigOfHost2 = tester.nodeRepository().nodes().node(host2).get().ipConfig(); + + // Two hosts are removed + tester.nodeRepository().nodes().removeRecursively(host1); + tester.nodeRepository().nodes().removeRecursively(host2); + assertEquals(2, tester.nodeRepository().nodes().list(Node.State.deprovisioned).size()); + + // Host not rebuilding cannot be restored + try { + tester.nodeRepository().nodes().restore(host1, Agent.system, getClass().getSimpleName()); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + // Other host is restored + Node node = tester.nodeRepository().nodes().restore(host2, Agent.system, getClass().getSimpleName()); + assertSame(Node.State.provisioned, node.state()); + assertEquals("IP addresses are preserved", ipConfigOfHost2, node.ipConfig()); + assertFalse(node.status().wantToRetire()); + assertFalse(node.status().wantToRebuild()); + } + + @Test public void dirty_host_only_if_we_can_dirty_children() { NodeRepositoryTester tester = new NodeRepositoryTester(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 305f1b5952e..b761f743687 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -475,7 +475,7 @@ public class DynamicProvisioningMaintainerTest { Supplier<Node> nodeToRemove = () -> tester.nodeRepository().nodes().node(configNodes.childrenOf(hostnameToRemove).first().get().hostname()).get(); // Set want to retire and deprovision on host and children - tester.nodeRepository().nodes().deprovision(hostToRemove.get(), Agent.system, tester.clock().instant()); + tester.nodeRepository().nodes().deprovision(hostToRemove.get().hostname(), Agent.system, tester.clock().instant()); // Redeployment of config server application retires node tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 881646fa546..523ceeb94ce 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -228,7 +228,7 @@ public class NodeSerializerTest { } @Test - public void serialize_parentHostname() { + public void serialize_parent_hostname() { final String parentHostname = "parent.yahoo.com"; Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant) .parentHostname(parentHostname) @@ -307,6 +307,17 @@ public class NodeSerializerTest { } @Test + public void want_to_rebuild() { + Node node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(createNode())); + assertFalse(node.status().wantToRebuild()); + node = node.with(node.status().withWantToRetire(true, false, true)); + node = nodeSerializer.fromJson(State.active, nodeSerializer.toJson(node)); + assertTrue(node.status().wantToRetire()); + assertFalse(node.status().wantToDeprovision()); + assertTrue(node.status().wantToRebuild()); + } + + @Test public void vespa_version_serialization() { String nodeWithWantedVespaVersion = "{\n" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 9ac85b9a99c..e0715702722 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -221,6 +221,9 @@ public class NodesV2ApiTest { // Make sure that wantToRetire is applied recursively, but wantToDeprovision isn't tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), "\"wantToRetire\":true,\"preferToRetire\":false,\"wantToDeprovision\":false,"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToRebuild\": true, \"wantToRetire\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", @@ -234,7 +237,16 @@ public class NodesV2ApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); - // move the docker host to deprovisioned + // move a host marked as wantToRebuild to deprovisioned + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed dockerhost1.yahoo.com\"}"); + // ... and then restore it + assertResponse(new Request("http://localhost:8080/nodes/v2/state/provisioned/dockerhost1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved dockerhost1.yahoo.com to provisioned\"}"); + + // move a host to deprovisioned assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed dockerhost1.yahoo.com\"}"); @@ -242,8 +254,6 @@ public class NodesV2ApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", new byte[0], Request.Method.DELETE), "{\"message\":\"Permanently removed dockerhost1.yahoo.com\"}"); - - } @Test diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index 2a6e6793bcd..bbf1a1a251e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -112,6 +112,11 @@ public class LambdaFunctionNode extends CompositeNode { if ( ! (node.children().get(0) instanceof ReferenceNode) || ! (node.children().get(1) instanceof ReferenceNode)) { return Optional.empty(); } + var lhs = (ReferenceNode) node.children().get(0); + var rhs = (ReferenceNode) node.children().get(1); + if (! lhs.getName().equals(arguments.get(0)) || ! rhs.getName().equals(arguments.get(1))) { + return Optional.empty(); + } if (node.operators().size() != 1) { return Optional.empty(); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 5c4840a555d..e1daf8c8fe2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -237,6 +237,8 @@ public class EvaluationTestCase { // tensor join tester.assertEvaluates("{ {x:0,y:0}:15, {x:1,y:0}:35 }", "join(tensor0, tensor1, f(x,y) (x*y))", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:6, {x:1,y:0}:14 }", "join(tensor0, tensor1, f(x,y) (x+x))", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0}:2, {x:1}:-3 }", "join(tensor0, tensor1, f(x,y) (y-x))", "{ {x:0}:3, {x:1}:7 }", "{ {x:0}:5, {x:1}:4 }"); // -- join composites tester.assertEvaluates("{ }", "tensor0 * tensor0", "{}"); tester.assertEvaluates("{{x:0,y:0,z:0}:0.0}", "( tensor0 * tensor1 ) * ( tensor2 * tensor1 )", diff --git a/searchlib/src/tests/attribute/.gitignore b/searchlib/src/tests/attribute/.gitignore index 732912ab981..d5747ffc3ff 100644 --- a/searchlib/src/tests/attribute/.gitignore +++ b/searchlib/src/tests/attribute/.gitignore @@ -1,5 +1,6 @@ *.dat *.idx +*.udat *.weight .depend Makefile diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 61d5b3795e4..7677a71ba3a 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/attribute/attributememoryfilebufferwriter.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributesaver.h> +#include <vespa/searchlib/attribute/i_enum_store_dictionary.h> #include <vespa/searchlib/queryeval/executeinfo.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> @@ -530,9 +531,28 @@ EnumeratedSaveTest::saveMemDuringCompaction(AttributeVector &v) void EnumeratedSaveTest::checkMem(AttributeVector &v, const MemAttr &e) { - MemAttr m; - EXPECT_TRUE(v.save(m, v.getBaseFileName())); - ASSERT_TRUE(m == e); + auto *esb = v.getEnumStoreBase(); + if (esb == nullptr || esb->get_dictionary().get_has_btree_dictionary()) { + MemAttr m; + EXPECT_TRUE(v.save(m, v.getBaseFileName())); + ASSERT_TRUE(m == e); + } else { + // Save without sorting unique values, load into temporary + // attribute vector with sorted dictionary and save again + // to verify data. + search::AttributeMemorySaveTarget ms; + search::TuneFileAttributes tune; + search::index::DummyFileHeaderContext fileHeaderContext; + EXPECT_TRUE(v.save(ms, "convert")); + EXPECT_TRUE(ms.writeToFile(tune, fileHeaderContext)); + auto cfg = v.getConfig(); + cfg.set_dictionary_config(search::DictionaryConfig(search::DictionaryConfig::Type::BTREE)); + auto v2 = AttributeFactory::createAttribute("convert", cfg); + EXPECT_TRUE(v2->load()); + MemAttr m2; + EXPECT_TRUE(v2->save(m2, v.getBaseFileName())); + ASSERT_TRUE(m2 == e); + } } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp index c335c2064f1..56cca654b96 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp @@ -9,10 +9,13 @@ namespace search::enumstore { EnumeratedLoaderBase::EnumeratedLoaderBase(IEnumStore& store) : _store(store), - _indexes() + _indexes(), + _enum_value_remapping() { } +EnumeratedLoaderBase::~EnumeratedLoaderBase() = default; + void EnumeratedLoaderBase::load_unique_values(const void* src, size_t available) { @@ -32,6 +35,41 @@ EnumeratedLoaderBase::free_unused_values() _store.free_unused_values(); } +void +EnumeratedLoaderBase::build_enum_value_remapping() +{ + if (!_store.get_dictionary().get_has_btree_dictionary() || _indexes.size() < 2u) { + return; // No need for unique values to be sorted + } + auto comp_up = _store.allocate_comparator(); + auto& comp = *comp_up; + if (std::is_sorted(_indexes.begin(), _indexes.end(), [&comp](Index lhs, Index rhs) { return !comp.less(rhs, lhs); })) { + return; // Unique values are already sorted + } + vespalib::Array<std::pair<Index, uint32_t>> sortdata; + uint32_t enum_value = 0; + sortdata.reserve(_indexes.size()); + for (auto index : _indexes) { + sortdata.push_back(std::make_pair(index, enum_value)); + ++enum_value; + } + std::sort(sortdata.begin(), sortdata.end(), [&comp](auto lhs, auto rhs) { return comp.less(lhs.first, rhs.first); }); + _enum_value_remapping.resize(_indexes.size()); + enum_value = 0; + for (auto &entry : sortdata) { + _indexes[enum_value] = entry.first; + _enum_value_remapping[entry.second] = enum_value; + ++enum_value; + } + assert(std::is_sorted(_indexes.begin(), _indexes.end(), [&comp](Index lhs, Index rhs) { return !comp.less(rhs, lhs); })); +} + +void +EnumeratedLoaderBase::free_enum_value_remapping() +{ + EnumVector().swap(_enum_value_remapping); +} + EnumeratedLoader::EnumeratedLoader(IEnumStore& store) : EnumeratedLoaderBase(store), _enums_histogram() diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h index 87705681dcf..4fa63646ed3 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h @@ -16,12 +16,17 @@ class EnumeratedLoaderBase { protected: IEnumStore& _store; IndexVector _indexes; + EnumVector _enum_value_remapping; // Empty if saved unique values are sorted. void release_enum_indexes(); public: EnumeratedLoaderBase(IEnumStore& store); + ~EnumeratedLoaderBase(); const IndexVector& get_enum_indexes() const { return _indexes; } + const EnumVector& get_enum_value_remapping() const noexcept { return _enum_value_remapping; } void load_unique_values(const void* src, size_t available); + void build_enum_value_remapping(); + void free_enum_value_remapping(); void free_unused_values(); }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 6d77295db08..05f7b6d0fe0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -208,6 +208,7 @@ public: _store.get_allocator().get_data_store().inc_compaction_count(); } std::unique_ptr<Enumerator> make_enumerator() const override; + std::unique_ptr<vespalib::datastore::EntryComparator> allocate_comparator() const override; }; std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 357026ab944..8ec61df5ac8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -61,13 +61,7 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& return -1; } const auto* value = static_cast<const EntryType*>(src); - Index prev_idx = idx; idx = _store.get_allocator().allocate(*value); - - if (prev_idx.valid()) { - auto cmp = make_comparator(*value); - assert(cmp.less(prev_idx, Index())); - } return sizeof(EntryType); } @@ -261,7 +255,14 @@ template <typename EntryT> std::unique_ptr<IEnumStore::Enumerator> EnumStoreT<EntryT>::make_enumerator() const { - return std::make_unique<Enumerator>(*_dict, _store.get_data_store()); + return std::make_unique<Enumerator>(*_dict, _store.get_data_store(), false); +} + +template <typename EntryT> +std::unique_ptr<vespalib::datastore::EntryComparator> +EnumStoreT<EntryT>::allocate_comparator() const +{ + return std::make_unique<ComparatorType>(_store.get_data_store()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index 895e6a6f4c0..ab2a588a53d 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -94,7 +94,7 @@ FlagAttributeT<B>::onLoadEnumerated(ReaderBase &attrReader) vespalib::ConstArrayRef<TT> map(reinterpret_cast<const TT *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(TT)); SaveBits<FlagAttributeT<B>, TT> saver(map, *this); - uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, saver); + uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, vespalib::ConstArrayRef<uint32_t>(), saver); this->checkSetMaxValueCount(maxvc); return true; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 321459078b9..55cd4f88c25 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -79,6 +79,7 @@ public: } virtual std::unique_ptr<Enumerator> make_enumerator() const = 0; + virtual std::unique_ptr<vespalib::datastore::EntryComparator> allocate_comparator() const = 0; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index 701c8eaf702..5e9bc80f46a 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -82,11 +82,11 @@ LoadUtils::loadUDAT(const AttributeVector& attr) #define INSTANTIATE_ARRAY(ValueType, Saver) \ -template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, Saver) +template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_WSET(ValueType, Saver) \ -template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, Saver) +template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE(ValueType, Saver) \ -template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, Saver) +template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, Saver) \ INSTANTIATE_SINGLE(ValueType, Saver); \ diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index cd9d98084d5..6833ab6b0b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -42,6 +42,7 @@ uint32_t loadFromEnumeratedMultiValue(MvMapping &mapping, ReaderBase &attrReader, vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); /** @@ -54,6 +55,7 @@ loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); } diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 61d56cfa4d9..4f856314997 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -12,6 +12,7 @@ uint32_t loadFromEnumeratedMultiValue(MvMapping & mapping, ReaderBase & attrReader, vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { mapping.prepareLoadFromMultiValue(); @@ -30,6 +31,9 @@ loadFromEnumeratedMultiValue(MvMapping & mapping, for (uint32_t vci = 0; vci < valueCount; ++vci) { uint32_t enumValue = attrReader.getNextEnum(); assert(enumValue < enumValueToValueMap.size()); + if (!enum_value_remapping.empty()) { + enumValue = enum_value_remapping[enumValue]; + } int32_t weight = MultiValueType::_hasWeight ? attrReader.getNextWeight() : 1; indices.emplace_back(enumValueToValueMap[enumValue], weight); saver.save(enumValue, doc, weight); @@ -51,6 +55,7 @@ loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap, + vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { uint32_t numDocs = attrReader.getEnumCount(); @@ -60,6 +65,9 @@ loadFromEnumeratedSingleValue(Vector &vector, for (uint32_t doc = 0; doc < numDocs; ++doc) { uint32_t enumValue = attrReader.getNextEnum(); assert(enumValue < enumValueToValueMap.size()); + if (!enum_value_remapping.empty()) { + enumValue = enum_value_remapping[enumValue]; + } vector.push_back(enumValueToValueMap[enumValue]); saver.save(enumValue, doc, 1); } diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index bd3a8adb56e..d320ecfaa85 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -99,7 +99,9 @@ MultiValueEnumAttribute<B, M>::load_enumerated_data(ReaderBase& attrReader, loader.reserve_loaded_enums(num_values); uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, vespalib::ConstArrayRef<EnumIndex>(loader.get_enum_indexes()), + loader.get_enum_value_remapping(), attribute::SaveLoadedEnum(loader.get_loaded_enums())); + loader.free_enum_value_remapping(); loader.sort_loaded_enums(); this->checkSetMaxValueCount(maxvc); } @@ -112,7 +114,9 @@ MultiValueEnumAttribute<B, M>::load_enumerated_data(ReaderBase& attrReader, loader.allocate_enums_histogram(); uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, vespalib::ConstArrayRef<EnumIndex>(loader.get_enum_indexes()), + loader.get_enum_value_remapping(), attribute::SaveEnumHist(loader.get_enums_histogram())); + loader.free_enum_value_remapping(); loader.set_ref_counts(); loader.build_dictionary(); loader.free_unused_values(); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 3ca7423c38c..b0aa2dcb6c0 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -120,7 +120,7 @@ MultiValueNumericAttribute<B, M>::onLoadEnumerated(ReaderBase & attrReader) auto udatBuffer = attribute::LoadUtils::loadUDAT(*this); assert((udatBuffer->size() % sizeof(T)) == 0); vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(T)); - uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, attribute::NoSaveLoadedEnum()); + uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, vespalib::ConstArrayRef<uint32_t>(), attribute::NoSaveLoadedEnum()); this->checkSetMaxValueCount(maxvc); return true; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index e17d41a5521..fffdbcde1bb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -68,6 +68,7 @@ MultiValueNumericEnumAttribute<B, M>::onLoadEnumerated(ReaderBase &attrReader) if (this->hasPostings()) { auto loader = this->getEnumStore().make_enumerated_postings_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); this->load_enumerated_data(attrReader, loader, numValues); if (numDocs > 0) { this->onAddDoc(numDocs - 1); @@ -76,6 +77,7 @@ MultiValueNumericEnumAttribute<B, M>::onLoadEnumerated(ReaderBase &attrReader) } else { auto loader = this->getEnumStore().make_enumerated_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); this->load_enumerated_data(attrReader, loader); } return true; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index fd4b0365ca1..2af5bfdf225 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -57,7 +57,7 @@ ReferenceAttribute::~ReferenceAttribute() _referenceMappings.clearBuilder(); incGeneration(); // Force freeze const auto &store = _store; - const auto enumerator = _store.getEnumerator(); + const auto enumerator = _store.getEnumerator(true); enumerator.foreach_key([&store,this](EntryRef ref) { const Reference &entry = store.get(ref); _referenceMappings.clearMapping(entry); diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index d3c71796838..aa76967e4ed 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -21,7 +21,7 @@ ReferenceAttributeSaver(GenerationHandler::Guard &&guard, : AttributeSaver(std::move(guard), header), _indices(std::move(indices)), _store(store), - _enumerator(store.getEnumerator()) + _enumerator(store.getEnumerator(true)) { } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 4d91c60ef4e..96dda48c043 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -208,7 +208,9 @@ SingleValueEnumAttribute<B>::load_enumerated_data(ReaderBase& attrReader, getGenerationHolder(), attrReader, loader.get_enum_indexes(), + loader.get_enum_value_remapping(), attribute::SaveLoadedEnum(loader.get_loaded_enums())); + loader.free_enum_value_remapping(); loader.sort_loaded_enums(); } @@ -222,7 +224,9 @@ SingleValueEnumAttribute<B>::load_enumerated_data(ReaderBase& attrReader, getGenerationHolder(), attrReader, loader.get_enum_indexes(), + loader.get_enum_value_remapping(), attribute::SaveEnumHist(loader.get_enums_histogram())); + loader.free_enum_value_remapping(); loader.set_ref_counts(); loader.build_dictionary(); loader.free_unused_values(); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 681c2af1f07..fd913f34c3a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -119,7 +119,7 @@ SingleValueNumericAttribute<B>::onLoadEnumerated(ReaderBase &attrReader) vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(T)); attribute::loadFromEnumeratedSingleValue(_data, getGenerationHolder(), attrReader, - map, attribute::NoSaveLoadedEnum()); + map, vespalib::ConstArrayRef<uint32_t>(), attribute::NoSaveLoadedEnum()); return true; } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index 5fb587c908e..dc1a6b8f278 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -90,6 +90,7 @@ SingleValueNumericEnumAttribute<B>::onLoadEnumerated(ReaderBase &attrReader) if (this->hasPostings()) { auto loader = this->getEnumStore().make_enumerated_postings_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); this->load_enumerated_data(attrReader, loader, numValues); if (numDocs > 0) { this->onAddDoc(numDocs - 1); @@ -98,6 +99,7 @@ SingleValueNumericEnumAttribute<B>::onLoadEnumerated(ReaderBase &attrReader) } else { auto loader = this->getEnumStore().make_enumerated_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); this->load_enumerated_data(attrReader, loader); } return true; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 56a644a68b1..a308fc06af0 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -308,6 +308,7 @@ StringAttribute::onLoadEnumerated(ReaderBase &attrReader) if (hasPostings()) { auto loader = this->getEnumStoreBase()->make_enumerated_postings_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); load_enumerated_data(attrReader, loader, numValues); if (numDocs > 0) { onAddDoc(numDocs - 1); @@ -316,6 +317,7 @@ StringAttribute::onLoadEnumerated(ReaderBase &attrReader) } else { auto loader = this->getEnumStoreBase()->make_enumerated_loader(); loader.load_unique_values(udatBuffer->buffer(), udatBuffer->size()); + loader.build_enum_value_remapping(); load_enumerated_data(attrReader, loader); } return true; diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp index 9f11b96e672..da9bc1284fa 100644 --- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp @@ -50,6 +50,7 @@ struct DataStoreShardedHashTest : public ::testing::Test void read_work(uint32_t cnt); void read_work(); void write_work(uint32_t cnt); + void populate_sample_data(); }; @@ -173,6 +174,13 @@ DataStoreShardedHashTest::write_work(uint32_t cnt) LOG(info, "done %u write work", cnt); } +void +DataStoreShardedHashTest::populate_sample_data() +{ + for (uint32_t i = 0; i < 50; ++i) { + insert(i); + } +} TEST_F(DataStoreShardedHashTest, single_threaded_reader_without_updates) { @@ -216,4 +224,57 @@ TEST_F(DataStoreShardedHashTest, memory_usage_is_reported) EXPECT_LT(0, usage.allocatedBytesOnHold()); } +TEST_F(DataStoreShardedHashTest, foreach_key_works) +{ + populate_sample_data(); + std::vector<uint32_t> keys; + _hash_map.foreach_key([this, &keys](EntryRef ref) { keys.emplace_back(_allocator.get_wrapped(ref).value()); }); + std::sort(keys.begin(), keys.end()); + EXPECT_EQ(50, keys.size()); + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_EQ(i, keys[i]); + } +} + +TEST_F(DataStoreShardedHashTest, move_keys_works) +{ + populate_sample_data(); + std::vector<EntryRef> refs; + _hash_map.foreach_key([&refs](EntryRef ref) { refs.emplace_back(ref); }); + std::vector<EntryRef> new_refs; + _hash_map.move_keys([this, &new_refs](EntryRef ref) { auto new_ref = _allocator.move(ref); _allocator.hold(ref); new_refs.emplace_back(new_ref); return new_ref; }); + std::vector<EntryRef> verify_new_refs; + _hash_map.foreach_key([&verify_new_refs](EntryRef ref) { verify_new_refs.emplace_back(ref); }); + EXPECT_EQ(50u, refs.size()); + EXPECT_NE(refs, new_refs); + EXPECT_EQ(new_refs, verify_new_refs); + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_NE(refs[i], new_refs[i]); + auto value = _allocator.get_wrapped(refs[i]).value(); + auto new_value = _allocator.get_wrapped(refs[i]).value(); + EXPECT_EQ(value, new_value); + } +} + +TEST_F(DataStoreShardedHashTest, normalize_values_works) +{ + populate_sample_data(); + for (uint32_t i = 0; i < 50; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + result->second.store_relaxed(EntryRef(i + 200)); + } + _hash_map.normalize_values([](EntryRef ref) noexcept { return EntryRef(ref.ref() + 300); }); + for (uint32_t i = 0; i < 50; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + ASSERT_EQ(i + 500, result->second.load_relaxed().ref()); + result->second.store_relaxed(EntryRef()); + } +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index e61713e40d8..d9b3d25a908 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -133,7 +133,7 @@ struct TestBase : public ::testing::Test { } size_t entrySize() const { return sizeof(ValueType); } auto getBuilder(uint32_t uniqueValuesHint) { return store.getBuilder(uniqueValuesHint); } - auto getEnumerator() { return store.getEnumerator(); } + auto getEnumerator(bool sort_unique_values) { return store.getEnumerator(sort_unique_values); } size_t get_reserved(EntryRef ref) { return store.bufferState(ref).getTypeHandler()->getReservedElements(getBufferId(ref)); } @@ -404,7 +404,7 @@ TYPED_TEST(TestBase, store_can_be_enumerated) this->remove(this->add(this->values()[2])); this->trimHoldLists(); - auto enumerator = this->getEnumerator(); + auto enumerator = this->getEnumerator(true); std::vector<uint32_t> refs; enumerator.foreach_key([&](EntryRef ref) { refs.push_back(ref.ref()); }); std::vector<uint32_t> expRefs; @@ -445,7 +445,7 @@ TEST_F(DoubleTest, nan_is_handled) EXPECT_FALSE(std::signbit(store.get(refs[2]))); EXPECT_TRUE(std::isinf(store.get(refs[3]))); EXPECT_TRUE(std::signbit(store.get(refs[3]))); - auto enumerator = getEnumerator(); + auto enumerator = getEnumerator(true); enumerator.enumerateValues(); std::vector<uint32_t> enumerated; for (auto &ref : refs) { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index 565c1ceee61..d0a12ddd290 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -76,7 +76,7 @@ public: uint32_t getNumUniques() const; Builder getBuilder(uint32_t uniqueValuesHint); - Enumerator getEnumerator() const; + Enumerator getEnumerator(bool sort_unique_values) const; // Should only be used for unit testing const BufferState &bufferState(EntryRef ref) const; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index c5e4eb0d6fd..a883b2351de 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -229,9 +229,9 @@ UniqueStore<EntryT, RefT, Compare, Allocator>::getBuilder(uint32_t uniqueValuesH template <typename EntryT, typename RefT, typename Compare, typename Allocator> typename UniqueStore<EntryT, RefT, Compare, Allocator>::Enumerator -UniqueStore<EntryT, RefT, Compare, Allocator>::getEnumerator() const +UniqueStore<EntryT, RefT, Compare, Allocator>::getEnumerator(bool sort_unique_values) const { - return Enumerator(*_dict, _store); + return Enumerator(*_dict, _store, sort_unique_values); } template <typename EntryT, typename RefT, typename Compare, typename Allocator> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h index be591649310..d7ea449754c 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h @@ -31,7 +31,7 @@ private: void allocate_enum_values(); public: - UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store); + UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store, bool sort_unique_values); ~UniqueStoreEnumerator(); void enumerateValue(EntryRef ref); void enumerateValues(); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index 7a43b16e66a..378fc54750d 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -9,14 +9,16 @@ namespace vespalib::datastore { template <typename RefT> -UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store) +UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store, bool sort_unique_values) : _dict_snapshot(dict.get_read_snapshot()), _store(store), _enumValues(), _next_enum_val(1) { _dict_snapshot->fill(); - _dict_snapshot->sort(); + if (sort_unique_values) { + _dict_snapshot->sort(); + } allocate_enum_values(); } |