diff options
37 files changed, 265 insertions, 282 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index b50d8ce2e7c..3e27cb4b6f6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -43,4 +43,7 @@ public interface ZoneApi { /** Returns the region name within the cloud, e.g. 'us-east-1' in AWS */ String getCloudNativeRegionName(); + /** Returns the availability zone within the cloud, e.g. 'use1-az2' in AWS */ + default String getCloudNativeAvailabilityZone() { throw new UnsupportedOperationException(); } + } diff --git a/config/src/vespa/config/frt/frtconnection.cpp b/config/src/vespa/config/frt/frtconnection.cpp index ff2a82f855b..4f1bf1c280c 100644 --- a/config/src/vespa/config/frt/frtconnection.cpp +++ b/config/src/vespa/config/frt/frtconnection.cpp @@ -104,7 +104,7 @@ void FRTConnection::calculateSuspension(ErrorType type) switch(type) { case TRANSIENT: delay = std::min(MAX_DELAY_MULTIPLIER, ++_transientFailures) * _transientDelay; - LOG(warning, "Connection to %s failed or timed out", _address.c_str()); + LOG(debug, "Connection to %s failed or timed out", _address.c_str()); break; case FATAL: delay = std::min(MAX_DELAY_MULTIPLIER, ++_fatalFailures) * _fatalDelay; @@ -112,7 +112,7 @@ void FRTConnection::calculateSuspension(ErrorType type) } _suspendedUntil = now + delay; if (_suspendWarned < (now - WARN_INTERVAL)) { - LOG(warning, "FRT Connection %s suspended until %s", _address.c_str(), vespalib::to_string(to_utc(_suspendedUntil)).c_str()); + LOG(debug, "FRT Connection %s suspended until %s", _address.c_str(), vespalib::to_string(to_utc(_suspendedUntil)).c_str()); _suspendWarned = now; } } diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index 66750b2943d..ac7b3d24d08 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.request; +import com.yahoo.concurrent.CopyOnWriteHashMap; + import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import static com.yahoo.text.Lowercase.toLowerCase; @@ -36,7 +39,8 @@ public final class CompoundName { /** The empty compound */ public static final CompoundName empty = new CompoundName(""); - + private static final Map<String, CompoundName> cache = new CopyOnWriteHashMap<>(); + private static final int MAX_CACHE_SIZE = 10_000; /** * Constructs this from a string which may contains dot-separated components * @@ -298,7 +302,20 @@ public final class CompoundName { return b.length()==0 ? "" : b.substring(0, b.length()-1); } - public static CompoundName from(String name) { return new CompoundName(name); } + /** + * Creates a CompoundName from a string, possibly reusing from cache. + * Prefer over constructing on the fly. + **/ + public static CompoundName from(String name) { + CompoundName found = cache.get(name); + if (found != null) return found; + + CompoundName compound = new CompoundName(name); + if (cache.size() < MAX_CACHE_SIZE) { + cache.put(name, compound); + } + return compound; + } private static class ImmutableArrayList extends AbstractList<String> { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java index 10b2cf0fdc7..409b502f086 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java @@ -25,7 +25,7 @@ public class JSONDebugSearcher extends Searcher { public static final String STRUCT_FIELD = "Structured data field (as json): "; public static final String FEATURE_FIELD = "Feature data field (as json): "; - private static CompoundName PROPERTYNAME = new CompoundName("dumpjson"); + private static final CompoundName PROPERTYNAME = new CompoundName("dumpjson"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index cff43e07d70..39fd372d2a7 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -426,28 +426,36 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * dependent objects for the appropriate subset of the given property values */ private void setFieldsFrom(Properties properties, Map<String, String> context) { - setFrom(CompoundName.empty, properties, Query.getArgumentType(), context); + setFrom("", properties, Query.getArgumentType(), context); + } + + private static String append(String a, String b) { + if (a.isEmpty()) return b; + if (b.isEmpty()) return a; + return a + "." + b; } /** * For each field in the given query profile type, take the corresponding value from originalProperties * (if any) set it to properties(), recursively. */ - private void setFrom(CompoundName prefix, Properties originalProperties, QueryProfileType arguments, Map<String, String> context) { - prefix = prefix.append(getPrefix(arguments)); + private void setFrom(String prefix, Properties originalProperties, QueryProfileType arguments, Map<String, String> context) { + prefix = append(prefix, getPrefix(arguments).toString()); for (FieldDescription field : arguments.fields().values()) { if (field.getType() == FieldType.genericQueryProfileType) { // Generic map - CompoundName fullName = prefix.append(field.getCompoundName()); - for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName, context).entrySet()) { - properties().set(fullName.append(entry.getKey()), entry.getValue(), context); + String fullName = append(prefix, field.getCompoundName().toString()); + for (Map.Entry<String, Object> entry : originalProperties.listProperties(CompoundName.from(fullName), context).entrySet()) { + properties().set(CompoundName.from(append(fullName, entry.getKey())), entry.getValue(), context); } } else if (field.getType() instanceof QueryProfileFieldType) { // Nested arguments setFrom(prefix, originalProperties, ((QueryProfileFieldType)field.getType()).getQueryProfileType(), context); } else { - CompoundName fullName = prefix.append(field.getCompoundName()); + CompoundName fullName = prefix.isEmpty() + ? field.getCompoundName() + : CompoundName.from(append(prefix, field.getCompoundName().toString())); Object value = originalProperties.get(fullName, context); if (value != null) { properties().set(fullName, value, context); @@ -458,14 +466,15 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Calls properties.set on all entries in requestMap */ private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties, boolean ignoreSelect) { - for (var entry : requestMap.entrySet()) { + var entrySet = requestMap.entrySet(); + for (var entry : entrySet) { if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; if (RankFeatures.isFeatureName(entry.getKey())) continue; // Set these last - properties.set(entry.getKey(), entry.getValue(), requestMap); + properties.set(CompoundName.from(entry.getKey()), entry.getValue(), requestMap); } - for (var entry : requestMap.entrySet()) { + for (var entry : entrySet) { if ( ! RankFeatures.isFeatureName(entry.getKey())) continue; - properties.set(entry.getKey(), entry.getValue(), requestMap); + properties.set(CompoundName.from(entry.getKey()), entry.getValue(), requestMap); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index c4541fe9f58..cd4cb32df2e 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -486,7 +486,8 @@ public class QueryProfileTestCase { assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[]{"x1"}))); } - public void testSettingNonLeaf4b() { + @Test + void testSettingNonLeaf4b() { QueryProfile p = new QueryProfile("test"); p.setDimensions(new String[] {"x"}); p.set("a","a-value", (QueryProfileRegistry)null); @@ -648,6 +649,32 @@ public class QueryProfileTestCase { assertTrue(traceContains("foo: value", query)); } + @Test + void benchQueryCreation() throws InterruptedException { + QueryProfile p = new QueryProfile("test"); + p.setDimensions(new String[]{"x", "y"}); + p.set("clustering.something", "bar", null); + p.set("clustering.something", "bar", new String[]{"x1", "y1"}, null); + p.freeze(); + CompiledQueryProfile cqp = p.compile(null); + var httpRequest = HttpRequest.createTestRequest("?x=x1&y=y1&query=bar&clustering.timeline.kano=tur&" + + "clustering.enable=true&clustering.timeline.bucketspec=-" + + "7d/3h&clustering.timeline.tophit=false&clustering.timeli" + + "ne=true", Method.GET); + for (int i = 0; i < 30000; i++) { + Query q = new Query(httpRequest, cqp); + assertTrue(q.properties().getBoolean(CompoundName.from("clustering.timeline"), false)); + } + Thread.sleep(2000); + long start = System.nanoTime(); + for (int i = 0; i < 100000; i++) { + Query q = new Query(httpRequest, cqp); + assertTrue(q.properties().getBoolean(CompoundName.from("clustering.timeline"), false)); + } + long now = System.nanoTime(); + System.out.println("Duration = " + (now - start)/1_000_000 + " ms"); + } + // NB: NOT RECURSIVE private boolean traceContains(String string, Query query) { for (TraceNode node : query.getContext(true).getTrace().traceNode().children()) diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 559b814a265..748a20801d1 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -327,6 +327,14 @@ public class QueryTestCase { } @Test + void testBooleanParameterNoQueryProfile() { + QueryProfile profile = new QueryProfile("myProfile"); + Query query = new Query("/?query=something&ranking.softtimeout.enable=false"); + assertFalse(query.properties().getBoolean("ranking.softtimeout.enable")); + assertFalse(query.getRanking().getSoftTimeout().getEnable()); + } + + @Test void testQueryProfileSubstitution2() { QueryProfile profile = new QueryProfile("myProfile"); profile.set("model.language", "en-US", null); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 4956dd475de..6cdb197b307 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.net.URI; import java.time.Duration; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; /** @@ -50,6 +51,9 @@ public interface ZoneRegistry { /** Returns the default region for the given environment, if one is configured */ Optional<RegionName> getDefaultRegion(Environment environment); + /** Throws {@link NoSuchElementException} if there is no such zone (in the system). */ + ZoneApi get(ZoneId zoneId); + /** Returns the URI for the config server VIP in the given zone */ URI getConfigServerVipUri(ZoneId zoneId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 6c614738ddf..8a0e2d01d8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -27,6 +27,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.handler.metrics.JsonResponse; import com.yahoo.container.jdisc.EmptyResponse; @@ -1659,6 +1660,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { else { deploymentObject.setString("environment", deployment.zone().environment().value()); deploymentObject.setString("region", deployment.zone().region().value()); + addAvailabilityZone(deploymentObject, deployment.zone()); deploymentObject.setString("url", withPath(request.getUri().getPath() + "/instance/" + instance.name().value() + "/environment/" + deployment.zone().environment().value() + @@ -1755,6 +1757,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { deploymentObject.setString("environment", deployment.zone().environment().value()); deploymentObject.setString("region", deployment.zone().region().value()); deploymentObject.setString("instance", instance.id().instance().value()); // pointless + addAvailabilityZone(deploymentObject, deployment.zone()); deploymentObject.setString("url", withPath(request.getUri().getPath() + "/environment/" + deployment.zone().environment().value() + "/region/" + deployment.zone().region().value(), @@ -1837,6 +1840,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("instance", deploymentId.applicationId().instance().value()); // pointless response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); + addAvailabilityZone(response, deployment.zone()); var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); // Add zone endpoints @@ -3020,6 +3024,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new MessageResponse("All deployments removed"); } + private void addAvailabilityZone(Cursor object, ZoneId zoneId) { + ZoneApi zone = controller.zoneRegistry().get(zoneId); + if (!zone.getCloudName().equals(CloudName.AWS)) return; + object.setString("availabilityZone", zone.getCloudNativeAvailabilityZone()); + } + private ZoneId requireZone(String environment, String region) { return requireZone(ZoneId.from(environment, region)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 528ef6d6192..6fd44e09d8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -20,15 +20,17 @@ public class ZoneApiMock implements ZoneApi { private final ZoneId virtualId; private final CloudName cloudName; private final String cloudNativeRegionName; + private final String cloudNativeAvailabilityZone; public static Builder newBuilder() { return new Builder(); } - private ZoneApiMock(SystemName systemName, ZoneId id, ZoneId virtualId, CloudName cloudName, String cloudNativeRegionName) { + private ZoneApiMock(SystemName systemName, ZoneId id, ZoneId virtualId, CloudName cloudName, String cloudNativeRegionName, String cloudNativeAvailabilityZone) { this.systemName = systemName; this.id = id; this.virtualId = virtualId; this.cloudName = cloudName; this.cloudNativeRegionName = cloudNativeRegionName; + this.cloudNativeAvailabilityZone = cloudNativeAvailabilityZone; if (virtualId != null && virtualId.equals(id)) { throw new IllegalArgumentException("Virtual ID cannot be equal to zone ID: " + id); } @@ -64,6 +66,9 @@ public class ZoneApiMock implements ZoneApi { public String getCloudNativeRegionName() { return cloudNativeRegionName; } @Override + public String getCloudNativeAvailabilityZone() { return cloudNativeAvailabilityZone; } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; @@ -88,6 +93,7 @@ public class ZoneApiMock implements ZoneApi { private ZoneId virtualId = null; private CloudName cloudName = CloudName.DEFAULT; private String cloudNativeRegionName = id.region().value(); + private String cloudNativeAvailabilityZone = "az1"; public Builder with(ZoneId id) { this.id = id; @@ -124,8 +130,13 @@ public class ZoneApiMock implements ZoneApi { return this; } + public Builder withCloudNativeAvailabilityZone(String cloudAvailabilityZone) { + this.cloudNativeAvailabilityZone = cloudAvailabilityZone; + return this; + } + public ZoneApiMock build() { - return new ZoneApiMock(systemName, id, virtualId, cloudName, cloudNativeRegionName); + return new ZoneApiMock(systemName, id, virtualId, cloudName, cloudNativeRegionName, cloudNativeAvailabilityZone); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index e59c677d0fa..611f0bab904 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -62,9 +63,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry if (system.isPublic()) { this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), - ZoneApiMock.fromId("prod.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-eu-west-1a"), - ZoneApiMock.fromId("dev.aws-us-east-1c")); + ZoneApiMock.newBuilder().withId("prod.aws-us-east-1c").withCloud("aws").withCloudNativeAvailabilityZone("use1-az2").build(), + ZoneApiMock.newBuilder().withId("prod.aws-eu-west-1a").withCloud("aws").withCloudNativeAvailabilityZone("euw1-az3").build(), + ZoneApiMock.newBuilder().withId("dev.aws-us-east-1c").withCloud("aws").withCloudNativeAvailabilityZone("use1-az2").build()); setRoutingMethod(this.zones, RoutingMethod.exclusive); } else { this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), @@ -299,6 +300,14 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public ZoneApi get(ZoneId zoneId) { + return zones.stream() + .filter(zone -> zone.getId().equals(zoneId)) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No zone with id '" + zoneId + "'")); + } + + @Override public URI getMonitoringSystemUri(DeploymentId deploymentId) { return URI.create("http://monitoring-system.test/?environment=" + deploymentId.zoneId().environment().value() + "®ion=" + deploymentId.zoneId().region().value() + "&application=" + deploymentId.applicationId().toShortString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index 0490a9bdcc5..fe14d696011 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -43,10 +43,12 @@ public class ArchiveAccessMaintainerTest { assertEquals(new ArchiveAccess().withAWSRole(tenant1role), archiveService.authorizeAccessByTenantName.get(tenant1)); assertEquals(new ArchiveAccess().withAWSRole(tenant2role), archiveService.authorizeAccessByTenantName.get(tenant2)); + var zoneRegistry = tester.controller().zoneRegistry(); var expected = Map.of("archive.bucketCount", - tester.controller().zoneRegistry().zonesIncludingSystem().all().ids().stream() + zoneRegistry.zonesIncludingSystem().all().ids().stream() .collect(Collectors.toMap( - zone -> Map.of("zone", zone.value(), "cloud", "default"), + zone -> Map.of("zone", zone.value(), "cloud", + zoneRegistry.hasZone(zone) ? zoneRegistry.get(zone).getCloudName().value() : "default"), zone -> zone.equals(testZone) ? 1d : 0d))); assertEquals(expected, metric.metrics()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json index 4d0f1259c07..bb4136ed0ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-cloud.json @@ -4,6 +4,7 @@ "instance": "default", "environment": "prod", "region": "aws-us-east-1c", + "availabilityZone": "use1-az2", "endpoints": [ { "cluster": "default", @@ -30,7 +31,7 @@ }, "status": "complete", "quota": 1.304, - "activity": { }, + "activity": {}, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 131460b0384..426e577b2d2 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -64,7 +64,6 @@ vespa_define_module( src/tests/proton/attribute/attributes_state_explorer src/tests/proton/attribute/document_field_extractor src/tests/proton/attribute/document_field_populator - src/tests/proton/attribute/exclusive_attribute_read_accessor src/tests/proton/attribute/imported_attributes_context src/tests/proton/attribute/imported_attributes_repo src/tests/proton/bucketdb/bucketdb diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index c9f3a1c1de8..140012624c2 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -5,7 +5,6 @@ #include <vespa/searchcore/proton/attribute/attribute_manager_reconfig.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> -#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> @@ -755,15 +754,6 @@ TEST_F("require that we can call functions on all attributes via functor", EXPECT_EQUAL("a1,a2,a3", functor->getSortedNames()); } -TEST_F("require that we can acquire exclusive read access to attribute", Fixture) -{ - f.addAttribute("attr"); - ExclusiveAttributeReadAccessor::UP attrAccessor = f._m.getExclusiveReadAccessor("attr"); - ExclusiveAttributeReadAccessor::UP noneAccessor = f._m.getExclusiveReadAccessor("none"); - EXPECT_TRUE(attrAccessor.get() != nullptr); - EXPECT_TRUE(noneAccessor.get() == nullptr); -} - TEST_F("require that imported attributes are exposed via attribute context together with regular attributes", Fixture) { f.addAttribute("attr"); diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index bcd200cfd2f..aa4871d3a12 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -92,13 +92,13 @@ TEST_F(AttributesStateExplorerTest, require_that_attributes_are_exposed_as_child { StringVector children = _explorer.get_children_names(); std::sort(children.begin(), children.end()); - EXPECT_EQ(StringVector({"btree", "extra", "hash", "hybrid", "regular"}), children); + EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "regular"}), children); } TEST_F(AttributesStateExplorerTest, require_that_attributes_are_explorable) { EXPECT_TRUE(_explorer.get_child("regular").get() != nullptr); - EXPECT_TRUE(_explorer.get_child("extra").get() != nullptr); + EXPECT_TRUE(_explorer.get_child("extra").get() == nullptr); EXPECT_TRUE(_explorer.get_child("not").get() == nullptr); } diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore deleted file mode 100644 index f3666eecb6e..00000000000 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchcore_exclusive_attribute_read_accessor_test_app diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt deleted file mode 100644 index 981d2acd7c5..00000000000 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app TEST - SOURCES - exclusive_attribute_read_accessor_test.cpp - DEPENDS - searchcore_attribute - searchcore_pcommon -) -vespa_add_test(NAME searchcore_exclusive_attribute_read_accessor_test_app COMMAND searchcore_exclusive_attribute_read_accessor_test_app) diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp deleted file mode 100644 index 8b093be08b7..00000000000 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> -#include <vespa/searchcommon/attribute/config.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/vespalib/util/gate.h> - -using namespace proton; -using namespace search; -using namespace vespalib; - -using ReadGuard = ExclusiveAttributeReadAccessor::Guard; -VESPA_THREAD_STACK_TAG(test_executor) - -AttributeVector::SP -createAttribute() -{ - attribute::Config cfg(attribute::BasicType::INT32, attribute::CollectionType::SINGLE); - return search::AttributeFactory::createAttribute("myattr", cfg); -} - -struct Fixture -{ - AttributeVector::SP attribute; - std::unique_ptr<ISequencedTaskExecutor> writer; - ExclusiveAttributeReadAccessor accessor; - - Fixture() - : attribute(createAttribute()), - writer(SequencedTaskExecutor::create(test_executor, 1)), - accessor(attribute, *writer) - {} -}; - -TEST_F("require that attribute write thread is blocked while guard is held", Fixture) -{ - ReadGuard::UP guard = f.accessor.takeGuard(); - Gate gate; - f.writer->execute(f.writer->getExecutorIdFromName(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); - bool reachedZero = gate.await(100ms); - EXPECT_FALSE(reachedZero); - EXPECT_EQUAL(1u, gate.getCount()); - - guard.reset(); - gate.await(); - EXPECT_EQUAL(0u, gate.getCount()); -} - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 82bb188870f..70a91b418a9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_collection_spec.cpp attribute_config_inspector.cpp attribute_directory.cpp + attribute_executor.cpp attribute_factory.cpp attribute_initializer.cpp attribute_initializer_result.cpp @@ -30,7 +31,6 @@ vespa_add_library(searchcore_attribute STATIC document_field_extractor.cpp document_field_populator.cpp document_field_retriever.cpp - exclusive_attribute_read_accessor.cpp filter_attribute_manager.cpp flushableattribute.cpp imported_attributes_context.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.cpp new file mode 100644 index 00000000000..4d854bfdca1 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.cpp @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_executor.h" +#include "i_attribute_manager.h" +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/vespalib/util/isequencedtaskexecutor.h> +#include <future> + +using search::AttributeVector; + +namespace proton { + +AttributeExecutor::AttributeExecutor(std::shared_ptr<IAttributeManager> mgr, + std::shared_ptr<AttributeVector> attr) + : _mgr(std::move(mgr)), + _attr(std::move(attr)) +{ +} + +AttributeExecutor::~AttributeExecutor() = default; + +void +AttributeExecutor::run_sync(std::function<void()> task) const +{ + vespalib::string name = _attr->getNamePrefix(); + auto& writer = _mgr->getAttributeFieldWriter(); + std::promise<void> promise; + auto future = promise.get_future(); + auto id = writer.getExecutorIdFromName(name); + writer.execute(id, [&task, promise=std::move(promise)]() mutable { task(); promise.set_value(); }); + future.wait(); +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.h new file mode 100644 index 00000000000..4a13fe1d886 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_executor.h @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <functional> +#include <memory> + +namespace search { class AttributeVector; } + +namespace proton { + +struct IAttributeManager; + +/* + * Class for executing task in attribute vector write thread. + */ +class AttributeExecutor +{ +private: + std::shared_ptr<IAttributeManager> _mgr; + std::shared_ptr<search::AttributeVector> _attr; + +public: + AttributeExecutor(std::shared_ptr<IAttributeManager> mgr, + std::shared_ptr<search::AttributeVector> attr); + ~AttributeExecutor(); + void run_sync(std::function<void()> task) const; + const search::AttributeVector& get_attr() const noexcept { return *_attr; } +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp index b7d6cf9e87a..1ea02c729ff 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_manager_explorer.h" +#include "attribute_executor.h" #include "attribute_vector_explorer.h" #include <vespa/searchlib/attribute/attributevector.h> +using search::AttributeVector; using vespalib::slime::Inserter; namespace proton { @@ -25,8 +27,7 @@ AttributeManagerExplorer::get_state(const Inserter &inserter, bool full) const std::vector<vespalib::string> AttributeManagerExplorer::get_children_names() const { - std::vector<search::AttributeGuard> attributes; - _mgr->getAttributeListAll(attributes); + auto& attributes = _mgr->getWritableAttributes(); std::vector<vespalib::string> names; for (const auto &attr : attributes) { names.push_back(attr->getName()); @@ -37,11 +38,13 @@ AttributeManagerExplorer::get_children_names() const std::unique_ptr<vespalib::StateExplorer> AttributeManagerExplorer::get_child(vespalib::stringref name) const { - auto attr = _mgr->getExclusiveReadAccessor(name); - if (attr.get() != nullptr) { - return std::make_unique<AttributeVectorExplorer>(std::move(attr)); + auto guard = _mgr->getAttribute(name); + auto attr = guard ? guard->getSP() : std::shared_ptr<AttributeVector>(); + if (attr && _mgr->getWritableAttribute(name) != nullptr) { + auto executor = std::make_unique<AttributeExecutor>(_mgr, std::move(attr)); + return std::make_unique<AttributeVectorExplorer>(std::move(executor)); } - return std::unique_ptr<vespalib::StateExplorer>(); + return {}; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index c153f873480..6244bdbea33 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_vector_explorer.h" +#include "attribute_executor.h" #include <vespa/searchlib/attribute/i_enum_store.h> #include <vespa/searchlib/attribute/i_enum_store_dictionary.h> #include <vespa/searchlib/attribute/multi_value_mapping.h> @@ -14,9 +15,11 @@ using search::attribute::Status; using search::AddressSpaceUsage; using search::AttributeVector; using search::IEnumStore; +using search::StateExplorerUtils; using vespalib::AddressSpace; using vespalib::MemoryUsage; using search::attribute::MultiValueMappingBase; +using search::attribute::IAttributeVector; using search::attribute::IPostingListAttributeBase; using namespace vespalib::slime; @@ -25,26 +28,6 @@ namespace proton { namespace { void -convertStatusToSlime(const Status &status, Cursor &object) -{ - object.setLong("numDocs", status.getNumDocs()); - object.setLong("numValues", status.getNumValues()); - object.setLong("numUniqueValues", status.getNumUniqueValues()); - object.setLong("lastSerialNum", status.getLastSyncToken()); - object.setLong("updateCount", status.getUpdateCount()); - object.setLong("nonIdempotentUpdateCount", status.getNonIdempotentUpdateCount()); - object.setLong("bitVectors", status.getBitVectors()); - { - Cursor &memory = object.setObject("memoryUsage"); - memory.setLong("allocatedBytes", status.getAllocated()); - memory.setLong("usedBytes", status.getUsed()); - memory.setLong("deadBytes", status.getDead()); - memory.setLong("onHoldBytes", status.getOnHold()); - memory.setLong("onHoldBytesMax", status.getOnHoldMax()); - } -} - -void convertGenerationToSlime(const AttributeVector &attr, Cursor &object) { object.setLong("oldest_used", attr.get_oldest_used_generation()); @@ -71,7 +54,7 @@ convertAddressSpaceUsageToSlime(const AddressSpaceUsage &usage, Cursor &object) void convertMemoryUsageToSlime(const MemoryUsage &usage, Cursor &object) { - search::StateExplorerUtils::memory_usage_to_slime(usage, object); + StateExplorerUtils::memory_usage_to_slime(usage, object); } void @@ -116,20 +99,25 @@ convertPostingBaseToSlime(const IPostingListAttributeBase &postingBase, Cursor & } -AttributeVectorExplorer::AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute) - : _attribute(std::move(attribute)) +AttributeVectorExplorer::AttributeVectorExplorer(std::unique_ptr<AttributeExecutor> executor) + : _executor(std::move(executor)) { } void AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bool full) const { - ExclusiveAttributeReadAccessor::Guard::UP readGuard = _attribute->takeGuard(); - const AttributeVector &attr = readGuard->get(); + auto& attr = _executor->get_attr(); + _executor->run_sync([this, &attr, &inserter, full] { get_state_helper(attr, inserter, full); }); +} + +void +AttributeVectorExplorer::get_state_helper(const AttributeVector& attr, const vespalib::slime::Inserter &inserter, bool full) const +{ const Status &status = attr.getStatus(); Cursor &object = inserter.insertObject(); if (full) { - convertStatusToSlime(status, object.setObject("status")); + StateExplorerUtils::status_to_slime(status, object.setObject("status")); convertGenerationToSlime(attr, object.setObject("generation")); convertAddressSpaceUsageToSlime(attr.getAddressSpaceUsage(), object.setObject("addressSpaceUsage")); // TODO: Consider making enum store, multivalue mapping, posting list attribute and tensor attribute diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h index 204a81ed629..f7bcc98d321 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h @@ -2,21 +2,25 @@ #pragma once -#include "exclusive_attribute_read_accessor.h" #include <vespa/vespalib/net/http/state_explorer.h> +namespace search { class AttributeVector; } + namespace proton { +class AttributeExecutor; + /** * Class used to explore the state of an attribute vector. */ class AttributeVectorExplorer : public vespalib::StateExplorer { private: - ExclusiveAttributeReadAccessor::UP _attribute; + std::unique_ptr<const AttributeExecutor> _executor; + void get_state_helper(const search::AttributeVector& attr, const vespalib::slime::Inserter &inserter, bool full) const; public: - AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute); + AttributeVectorExplorer(std::unique_ptr<AttributeExecutor> executor); // Implements vespalib::StateExplorer void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 161a4294908..ee04f17d378 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -638,16 +638,6 @@ AttributeManager::asyncForAttribute(const vespalib::string &name, std::unique_pt [attr=std::move(attrsp), func=std::move(func)]() { (*func)(*attr); }); } -ExclusiveAttributeReadAccessor::UP -AttributeManager::getExclusiveReadAccessor(const vespalib::string &name) const -{ - AttributeVector::SP attribute = findAttribute(name); - if (attribute) { - return std::make_unique<ExclusiveAttributeReadAccessor>(attribute, _attributeFieldWriter); - } - return {}; -} - void AttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index c924c1c5b20..d6d0964f97e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -182,8 +182,6 @@ public: void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override; - ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; - void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp deleted file mode 100644 index ed72418ced4..00000000000 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "exclusive_attribute_read_accessor.h" -#include <vespa/vespalib/util/gate.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> - -namespace proton { - -using search::AttributeVector; -using vespalib::ISequencedTaskExecutor; -using vespalib::Executor; -using vespalib::Gate; - -using GateSP = std::shared_ptr<Gate>; - -ExclusiveAttributeReadAccessor::Guard::Guard(const AttributeVector &attribute, - const GateSP &exitGate) - : _attribute(attribute), - _exitGate(exitGate) -{ -} - -ExclusiveAttributeReadAccessor::Guard::~Guard() -{ - _exitGate->countDown(); -} - -ExclusiveAttributeReadAccessor:: -ExclusiveAttributeReadAccessor(const AttributeVector::SP &attribute, - ISequencedTaskExecutor &attributeFieldWriter) - : _attribute(attribute), - _attributeFieldWriter(attributeFieldWriter) -{ -} - -namespace { - -void -attributeWriteBlockingTask(AttributeVector::SP attribute, GateSP entranceGate, GateSP exitGate) -{ - attribute->commit(true); - entranceGate->countDown(); - exitGate->await(); -} - -} - -ExclusiveAttributeReadAccessor::Guard::UP -ExclusiveAttributeReadAccessor::takeGuard() -{ - GateSP entranceGate = std::make_shared<Gate>(); - GateSP exitGate = std::make_shared<Gate>(); - _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(_attribute->getNamePrefix()), - [this, entranceGate, exitGate]() { attributeWriteBlockingTask(_attribute, entranceGate, exitGate); }); - entranceGate->await(); - return std::make_unique<Guard>(*_attribute, exitGate); -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h deleted file mode 100644 index aa756764b1a..00000000000 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace search { - class AttributeVector; -} -namespace vespalib { - class Gate; - class ISequencedTaskExecutor; -} - -namespace proton { - -/** - * Class that provides exclusive read access to an attribute vector - * while the write thread for that attribute is blocked. - * - * The attribute write thread is blocked while a guard is held. - */ -class ExclusiveAttributeReadAccessor -{ -public: - class Guard - { - private: - const search::AttributeVector &_attribute; - std::shared_ptr<vespalib::Gate> _exitGate; - - public: - using UP = std::unique_ptr<Guard>; - Guard(const search::AttributeVector &attribute, - const std::shared_ptr<vespalib::Gate> &exitGate); - ~Guard(); - const search::AttributeVector &get() const { return _attribute; } - }; - -private: - using AttributeVectorSP = std::shared_ptr<search::AttributeVector>; - AttributeVectorSP _attribute; - vespalib::ISequencedTaskExecutor &_attributeFieldWriter; - -public: - using UP = std::unique_ptr<ExclusiveAttributeReadAccessor>; - - ExclusiveAttributeReadAccessor(const AttributeVectorSP &attribute, - vespalib::ISequencedTaskExecutor &attributeFieldWriter); - Guard::UP takeGuard(); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index 6f6cd01b4a4..6329c633727 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -232,12 +232,6 @@ FilterAttributeManager::asyncForAttribute(const vespalib::string &name, std::uni } -ExclusiveAttributeReadAccessor::UP -FilterAttributeManager::getExclusiveReadAccessor(const vespalib::string &name) const -{ - return (acceptAttribute(name)) ? _mgr->getExclusiveReadAccessor(name) : ExclusiveAttributeReadAccessor::UP(); -} - void FilterAttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributesRepo>) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index dff18290330..6d5ff682ca1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -52,7 +52,6 @@ public: const std::vector<search::AttributeVector *> & getWritableAttributes() const override; void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const override; void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const override; - ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override; std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index 437e7bd0208..5e60e950cc4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -2,7 +2,6 @@ #pragma once -#include "exclusive_attribute_read_accessor.h" #include "i_attribute_factory.h" #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> @@ -102,8 +101,6 @@ struct IAttributeManager : public search::IAttributeManager virtual void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const = 0; virtual void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor> func, OnDone onDone) const = 0; - virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0; - virtual void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) = 0; virtual const ImportedAttributesRepo *getImportedAttributes() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp index 5a1f0f42507..ffb9acd5501 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.cpp @@ -1,8 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_meta_store_explorer.h" +#include "documentmetastore.h" +#include <vespa/searchlib/util/state_explorer_utils.h> #include <vespa/vespalib/data/slime/cursor.h> +using search::StateExplorerUtils; +using search::attribute::Status; using vespalib::slime::Cursor; using vespalib::slime::Inserter; @@ -18,6 +22,11 @@ DocumentMetaStoreExplorer::get_state(const Inserter &inserter, bool full) const { Cursor &object = inserter.insertObject(); if (full) { + auto dms = dynamic_cast<const DocumentMetaStore*>(&_metaStore->get()); + if (dms != nullptr) { + const Status &status = dms->getStatus(); + StateExplorerUtils::status_to_slime(status, object.setObject("status")); + } search::LidUsageStats stats = _metaStore->get().getLidUsageStats(); object.setLong("usedLids", stats.getUsedLids()); object.setLong("activeLids", _metaStore->get().getNumActiveLids()); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index f85baa8c0ac..babecb6a77d 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -91,9 +91,6 @@ public: void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>) const override { } void asyncForEachAttribute(std::shared_ptr<IAttributeFunctor>, OnDone) const override { } - ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &) const override { - return ExclusiveAttributeReadAccessor::UP(); - } void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> importedAttributes) override { _importedAttributes = std::move(importedAttributes); } diff --git a/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp b/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp index 00e2e081302..d61737d2a5f 100644 --- a/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp +++ b/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp @@ -1,13 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "state_explorer_utils.h" +#include <vespa/searchcommon/attribute/status.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/util/memoryusage.h> +using search::attribute::Status; +using vespalib::MemoryUsage; +using vespalib::slime::Cursor; + namespace search { void -StateExplorerUtils::memory_usage_to_slime(const vespalib::MemoryUsage& usage, vespalib::slime::Cursor& object) +StateExplorerUtils::memory_usage_to_slime(const MemoryUsage& usage, Cursor& object) { object.setLong("allocated", usage.allocatedBytes()); object.setLong("used", usage.usedBytes()); @@ -15,5 +20,24 @@ StateExplorerUtils::memory_usage_to_slime(const vespalib::MemoryUsage& usage, ve object.setLong("onHold", usage.allocatedBytesOnHold()); } +void +StateExplorerUtils::status_to_slime(const Status &status, Cursor &object) +{ + object.setLong("numDocs", status.getNumDocs()); + object.setLong("numValues", status.getNumValues()); + object.setLong("numUniqueValues", status.getNumUniqueValues()); + object.setLong("lastSerialNum", status.getLastSyncToken()); + object.setLong("updateCount", status.getUpdateCount()); + object.setLong("nonIdempotentUpdateCount", status.getNonIdempotentUpdateCount()); + object.setLong("bitVectors", status.getBitVectors()); + { + Cursor &memory = object.setObject("memoryUsage"); + memory.setLong("allocatedBytes", status.getAllocated()); + memory.setLong("usedBytes", status.getUsed()); + memory.setLong("deadBytes", status.getDead()); + memory.setLong("onHoldBytes", status.getOnHold()); + memory.setLong("onHoldBytesMax", status.getOnHoldMax()); + } } +} diff --git a/searchlib/src/vespa/searchlib/util/state_explorer_utils.h b/searchlib/src/vespa/searchlib/util/state_explorer_utils.h index eebac033714..9a8d1a7d9db 100644 --- a/searchlib/src/vespa/searchlib/util/state_explorer_utils.h +++ b/searchlib/src/vespa/searchlib/util/state_explorer_utils.h @@ -2,6 +2,7 @@ #pragma once +namespace search::attribute { class Status; } namespace vespalib { class MemoryUsage; } namespace vespalib::slime { struct Cursor; } @@ -13,6 +14,7 @@ namespace search { class StateExplorerUtils { public: static void memory_usage_to_slime(const vespalib::MemoryUsage& usage, vespalib::slime::Cursor& object); + static void status_to_slime(const search::attribute::Status &status, vespalib::slime::Cursor &object); }; } diff --git a/vespalib/src/vespa/vespalib/util/optimized.h b/vespalib/src/vespa/vespalib/util/optimized.h index e78372b03ff..a85c1d18c00 100644 --- a/vespalib/src/vespa/vespalib/util/optimized.h +++ b/vespalib/src/vespa/vespalib/util/optimized.h @@ -31,7 +31,7 @@ public: * @fn int Optimized::msbIdx(uint32_t v) * @brief Quickly find most significant bit. * - * Finds the postion of the most significant '1'. + * Finds the position of the most significant '1'. * @param v is the value to search * @return index [0-31] of msb, 0 if none. **/ @@ -40,7 +40,7 @@ public: * @fn int Optimized::msbIdx(uint64_t v) * @brief Quickly find most significant bit. * - * Finds the postion of the most significant '1'. + * Finds the position of the most significant '1'. * @param v is the value to search * @return index [0-63] of msb, 0 if none. **/ @@ -49,7 +49,7 @@ public: * @fn int Optimized::lsbIdx(uint32_t v) * @brief Quickly find least significant bit. * - * Finds the postion of the least significant '1'. + * Finds the position of the least significant '1'. * @param v is the value to search * @return index [0-31] of lsb, 0 if none. **/ @@ -58,7 +58,7 @@ public: * @fn int Optimized::lsbIdx(uint64_t v) * @brief Quickly find least significant bit. * - * Finds the postion of the least significant '1'. + * Finds the position of the least significant '1'. * @param v is the value to search * @return index [0-63] of lsb, 0 if none. **/ |