diff options
23 files changed, 306 insertions, 64 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 14cd6da1c2e..672414f2d47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,18 +82,13 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int metricsproxyNumThreads() { return defaultPoolNumThreads(); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultPoolNumThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default int availableProcessors() { return 2; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean ignoreMergeQueueLimit() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @@ -101,17 +96,13 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default int distributorMergeBusyWait() { return 1; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}, removeAfter = "7.528.3") default boolean distributorEnhancedMaintenanceScheduling() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}, removeAfter = "7.528.3") default boolean asyncApplyBucketDiff() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double tlsSizeFraction() { return 0.02; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index f8a45a11b70..a9587da0f0b 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -200,11 +200,9 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { void setConfigAndGeneration(Long generation, boolean applyOnRestart, T config, PayloadChecksums payloadChecksums) { ConfigState<T> prev = this.config.get(); boolean configChanged = !Objects.equals(prev.getConfig(), config); - String message = "Config has changed unexpectedly for " + key + ", generation " + generation; if (configChanged) { - if (log.isLoggable(Level.FINE)) - message = message + ", config in state :" + prev.getConfig() + ", new config: " + config; - log.log(Level.WARNING, message); + log.log(Level.WARNING, "Config has changed unexpectedly for " + key + ", generation " + generation + + ", config in state :" + prev.getConfig() + ", new config: " + config); } this.config.set(new ConfigState<>(true, generation, applyOnRestart, configChanged, config, payloadChecksums)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 3f4ce7cc49b..bcc3b9b54c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -141,7 +141,7 @@ public abstract class LockedTenant { } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Optional.empty(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index b170a1cbb68..18c2dd49514 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -70,6 +70,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; @@ -1718,7 +1719,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { TenantName tenant = TenantName.from(tenantName); Inspector requestObject = toSlime(request.getData()).get(); controller.tenants().create(accessControlRequests.specification(tenant, requestObject), - accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); + accessControlRequests.credentials(tenant, requestObject, request.getJDiscRequest())); + if (controller.system().isPublic()) { + User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); + TenantInfo info = controller.tenants().require(tenant, CloudTenant.class) + .info() + .withContactName(user.name()) + .withContactEmail(user.email()); + // Store changes + controller.tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { + lockedTenant = lockedTenant.withInfo(info); + controller.tenants().store(lockedTenant); + }); + } return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index a6d061e7c80..2fd8026319b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -92,15 +92,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { Request request = new Request("http://localhost:8080" + path, data, method, principal); request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, roles)); if (user != null) { - Map<String, String> userAttributes = new HashMap<>(); - userAttributes.put("email", user.email()); - if (user.name() != null) - userAttributes.put("name", user.name()); - if (user.nickname() != null) - userAttributes.put("nickname", user.nickname()); - if (user.picture() != null) - userAttributes.put("picture", user.picture()); - request.getAttributes().put(User.ATTRIBUTE_NAME, Map.copyOf(userAttributes)); + request.getAttributes().put(User.ATTRIBUTE_NAME, user); } request.getHeaders().put("Content-Type", contentType); return request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 537f6c48bdf..a93e9f55e30 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -67,9 +67,16 @@ public class UserApiTest extends ControllerContainerCloudTest { tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .roles(operator) .principal("administrator@tenant") + .user(new User("administrator@tenant", "administrator", "admin", "picture")) .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); + // GET at tenant/info with contact information. + tester.assertResponse(request("/application/v4/tenant/my-tenant/info") + .roles(operator) + .principal("administrator@tenant"), + new File("tenant-info-after-created.json")); + // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), accessDenied, 403); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json new file mode 100644 index 00000000000..942b5c1db45 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json @@ -0,0 +1,8 @@ +{ + "name": "", + "email": "", + "website":"", + "invoiceEmail":"", + "contactName": "administrator", + "contactEmail": "administrator@tenant" +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 7cc1a51a114..54585767d51 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -1,6 +1,7 @@ { "tenant": "my-tenant", "type": "CLOUD", + "creator": "administrator@tenant", "pemDeveloperKeys": [ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 1662484ade8..1cd2fb41263 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -1,6 +1,7 @@ { "tenant": "my-tenant", "type": "CLOUD", + "creator": "administrator@tenant", "pemDeveloperKeys": [ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index e43ff26272a..73ee2ecaedd 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -3,9 +3,10 @@ package com.yahoo.document; import com.yahoo.compress.CompressionType; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.internal.GeoPosType; import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; @@ -97,11 +98,19 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } + boolean looksLikePosition(StructDataType type) { + var pos = PositionDataType.INSTANCE; + return type.getName().equals(pos.getName()) && type.getId() == pos.getId(); + } + private void startStructsAndDocs(DocumentmanagerConfig config) { for (var thisDataType : config.datatype()) { for (var o : thisDataType.structtype()) { int id = thisDataType.id(); StructDataType type = new StructDataType(id, o.name()); + if (usev8geopositions && looksLikePosition(type)) { + type = new GeoPosType(8); + } inProgress(type); configMap.remove(id); } @@ -198,6 +207,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (var struct : thisDataType.structtype()) { int id = thisDataType.id(); StructDataType type = (StructDataType) typesById.get(id); + if (type instanceof GeoPosType) { + continue; + } for (var parent : struct.inherits()) { var parentStruct = (StructDataType) typesByName.get(parent.name()); type.inherit(parentStruct); @@ -319,9 +331,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub private final DocumentTypeManager manager; } - private static class ApplyNewDoctypeConfig { + public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) { this.manager = manager; this.usev8geopositions = config.usev8geopositions(); @@ -379,7 +391,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub for (var typeconf : docTypeConfig.primitivetype()) { DataType type = manager.getDataType(typeconf.name()); if (! (type instanceof PrimitiveDataType)) { - throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type); + throw new IllegalArgumentException("Needed primitive type for '"+typeconf.name()+"' [idx "+typeconf.idx()+"] but got: "+type); } addNewType(typeconf.idx(), type); } @@ -411,10 +423,32 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } + private final Field POS_X = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_X); + private final Field POS_Y = PositionDataType.INSTANCE.getField(PositionDataType.FIELD_Y); + + boolean isPositionStruct(DocumentmanagerConfig.Doctype.Structtype cfg) { + if (! cfg.name().equals(PositionDataType.STRUCT_NAME)) return false; + if (! cfg.inherits().isEmpty()) return false; + if (cfg.field().size() != 2) return false; + var f0 = cfg.field(0); + var f1 = cfg.field(1); + if (! f0.name().equals(POS_X.getName())) return false; + if (! f1.name().equals(POS_Y.getName())) return false; + if (f0.internalid() != POS_X.getId()) return false; + if (f1.internalid() != POS_Y.getId()) return false; + if (typesByIdx.get(f0.type()) != POS_X.getDataType()) return false; + if (typesByIdx.get(f1.type()) != POS_Y.getDataType()) return false; + return true; + } + void createEmptyStructs() { String docName = docTypeConfig.name(); for (var typeconf : docTypeConfig.structtype()) { - addNewType(typeconf.idx(), new StructDataType(typeconf.name())); + if (usev8geopositions && isPositionStruct(typeconf)) { + addNewType(typeconf.idx(), new GeoPosType(8)); + } else { + addNewType(typeconf.idx(), new StructDataType(typeconf.name())); + } } } @@ -486,6 +520,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } void fillStructs() { for (var structCfg : docTypeConfig.structtype()) { + if (usev8geopositions && isPositionStruct(structCfg)) { + continue; + } int idx = structCfg.idx(); StructDataType type = (StructDataType) typesByIdx.get(idx); for (var parent : structCfg.inherits()) { @@ -541,11 +578,11 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } for (var docType : config.doctype()) { var docTypeData = inProgressById.get(docType.idx()); + docTypeData.createSimpleTypes(); docTypeData.createEmptyStructs(); docTypeData.initializeDocType(); docTypeData.createEmptyAnnotationTypes(); docTypeData.createFactories(); - docTypeData.createSimpleTypes(); } createComplexTypes(); for (var docType : config.doctype()) { diff --git a/document/src/main/java/com/yahoo/document/internal/GeoPosType.java b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java new file mode 100644 index 00000000000..2999f7506ee --- /dev/null +++ b/document/src/main/java/com/yahoo/document/internal/GeoPosType.java @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.document.internal; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.Struct; + +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; + +/** + * @author arnej + **/ +public final class GeoPosType extends StructDataType { + + private final boolean useV8json; + private static final Field F_X = new Field("x", DataType.INT); + private static final Field F_Y = new Field("y", DataType.INT); + + public GeoPosType(int vespaVersion) { + super("position"); + this.useV8json = (vespaVersion == 8); + assert(vespaVersion > 6); + assert(vespaVersion < 9); + addField(F_X); + addField(F_Y); + } + + public boolean renderJsonAsVespa8() { + return this.useV8json; + } + + public double getLatitude(Struct pos) { + assert(pos.getDataType() == this); + double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6; + return ns; + } + + public double getLongitude(Struct pos) { + assert(pos.getDataType() == this); + double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6; + return ew; + } + + private static final DecimalFormat degreeFmt; + + static { + degreeFmt = new DecimalFormat("0.0#####", DecimalFormatSymbols.getInstance(Locale.ENGLISH)); + degreeFmt.setMinimumIntegerDigits(1); + degreeFmt.setMinimumFractionDigits(1); + degreeFmt.setMaximumFractionDigits(6); + } + + static String fmtD(double degrees) { + return degreeFmt.format(degrees); + } + + public String fmtLatitude(Struct pos) { + assert(pos.getDataType() == this); + double ns = PositionDataType.getYValue(pos).getInteger() * 1.0e-6; + return fmtD(ns); + } + + public String fmtLongitude(Struct pos) { + assert(pos.getDataType() == this); + double ew = PositionDataType.getXValue(pos).getInteger() * 1.0e-6; + return fmtD(ew); + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 1d9fd3aa1ec..340bd542885 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -25,6 +25,7 @@ import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.TensorReader; import com.yahoo.document.json.readers.TensorRemoveUpdateReader; import com.yahoo.document.serialization.FieldWriter; @@ -153,12 +154,32 @@ public class JsonSerializationHelper { }); } + private static void serializeGeoPos(JsonGenerator generator, FieldBase field, Struct value, GeoPosType dataType) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> { + generator.writeStartObject(); + generator.writeFieldName("lat"); + generator.writeRawValue(dataType.fmtLatitude(value)); + generator.writeFieldName("lng"); + generator.writeRawValue(dataType.fmtLongitude(value)); + generator.writeEndObject(); + }); + } + public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { - if (value.getDataType() == PositionDataType.INSTANCE) { + DataType dt = value.getDataType(); + // TODO remove in Vespa 8: + if (dt == PositionDataType.INSTANCE) { serializeString(generator, field, PositionDataType.renderAsString(value)); return; } - + if (dt instanceof GeoPosType) { + var gpt = (GeoPosType)dt; + if (gpt.renderJsonAsVespa8()) { + serializeGeoPos(generator, field, value, gpt); + return; + } + } serializeStructuredField(fieldWriter, generator, field, value); } diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg index 6ceda63e606..a4cf62db0c7 100644 --- a/document/src/test/document/documentmanager.cfg +++ b/document/src/test/document/documentmanager.cfg @@ -1,3 +1,4 @@ +usev8geopositions true doctype[4] doctype[0].name "document" doctype[0].idx 1000 @@ -45,6 +46,14 @@ doctype[0].annotationtype[8].internalid 6 doctype[0].annotationtype[8].datatype 1004 doctype[0].structtype[0].idx 1001 doctype[0].structtype[0].name document.header +doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 1002 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 1002 doctype[1].name "foobar" doctype[1].idx 1014 doctype[1].inherits[0].idx 1000 diff --git a/document/src/test/document/documentmanager.testv8pos.cfg b/document/src/test/document/documentmanager.testv8pos.cfg new file mode 100644 index 00000000000..3f776748b79 --- /dev/null +++ b/document/src/test/document/documentmanager.testv8pos.cfg @@ -0,0 +1,31 @@ +usev8geopositions true +doctype[2] +doctype[0].name "document" +doctype[0].idx 1000 +doctype[0].contentstruct 1001 +doctype[0].primitivetype[0].idx 1002 +doctype[0].primitivetype[0].name "int" +doctype[0].structtype[0].idx 1001 +doctype[0].structtype[0].name document.header +doctype[0].structtype[1].idx 10010 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 1002 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 1002 +doctype[1].name "foobar" +doctype[1].idx 1014 +doctype[1].contentstruct 1015 +doctype[1].inherits[0].idx 1000 +doctype[1].arraytype[0].idx 1017 +doctype[1].arraytype[0].elementtype 10010 +doctype[1].structtype[0].idx 1015 +doctype[1].structtype[0].name foobar.header +doctype[1].structtype[0].field[0].name "simplepos" +doctype[1].structtype[0].field[0].internalid 1707020592 +doctype[1].structtype[0].field[0].type 10010 +doctype[1].structtype[0].field[1].name "arraypos" +doctype[1].structtype[0].field[1].internalid 1055920092 +doctype[1].structtype[0].field[1].type 1017 diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index 0aa5aec4b85..b89ed2b6b08 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -114,7 +114,7 @@ public class DocumentTypeManagerTestCase { assertSame(docType2, manager.getDocumentType(new DataTypeName("foo1"))); assertSame(docType3, manager.getDocumentType(new DataTypeName("foo2"))); assertSame(docType4, manager.getDocumentType(new DataTypeName("foo3"))); - + assertEquals(manager.getDocumentTypes().size(), 5); assertNotNull(manager.getDocumentTypes().get(new DataTypeName("document"))); assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), docType1); @@ -587,6 +587,18 @@ search annotationsimplicitstruct { assertFalse(docType.hasImportedField("a_missing_imported_field")); } + @Test + public void position_type_is_recognized_as_v8() { + var manager = DocumentTypeManager.fromFile("src/test/document/documentmanager.testv8pos.cfg"); + var docType = manager.getDocumentType("foobar"); + var simplepos = docType.getField("simplepos").getDataType(); + assertTrue(simplepos instanceof StructDataType); + var arraypos = docType.getField("arraypos").getDataType(); + assertTrue(arraypos instanceof ArrayDataType); + var array = (ArrayDataType) arraypos; + assertTrue(array.getNestedType() instanceof StructDataType); + } + // TODO test clone(). Also fieldSets not part of clone()..! // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this... diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 66ff7a7d4cd..ab4af5e722e 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -31,6 +31,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.document.serialization.DocumentSerializer; @@ -149,6 +150,7 @@ public class JsonReaderTestCase { DocumentType x = new DocumentType("testsinglepos"); DataType d = PositionDataType.INSTANCE; x.addField(new Field("singlepos", d)); + x.addField(new Field("geopos", new GeoPosType(8))); types.registerDocumentType(x); } { @@ -612,6 +614,43 @@ public class JsonReaderTestCase { } @Test + public void testPositionGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': 'N63.429722;E10.393333' }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + } + + @Test + public void testPositionOldGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': {'x':10393333,'y':63429722} }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + } + + @Test + public void testGeoPositionGeoPos() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'geopos': {'lat':63.429722,'lng':10.393333} }}")); + FieldValue f = doc.getFieldValue(doc.getField("geopos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + assertEquals(f.getDataType(), PositionDataType.INSTANCE); + assertEquals(PositionDataType.INSTANCE, f.getDataType()); + } + + @Test public void testPositionNegative() throws IOException { Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", " 'fields': {", diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 29703eadfce..7573aba519f 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.ReferenceFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.tensor.TensorType; @@ -93,6 +94,7 @@ public class JsonWriterTestCase { DocumentType x = new DocumentType("testmultipos"); DataType d = new ArrayDataType(PositionDataType.INSTANCE); x.addField(new Field("multipos", d)); + x.addField(new Field("geopos", new ArrayDataType(new GeoPosType(8)))); types.registerDocumentType(x); } @@ -100,6 +102,7 @@ public class JsonWriterTestCase { DocumentType x = new DocumentType("testsinglepos"); DataType d = PositionDataType.INSTANCE; x.addField(new Field("singlepos", d)); + x.addField(new Field("geopos", new GeoPosType(8))); types.registerDocumentType(x); } @@ -202,11 +205,13 @@ public class JsonWriterTestCase { @Test public void singlePosTest() throws IOException { roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }"); + roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"geopos\": { \"lat\": 60.222333, \"lng\": 10.12 } }"); } @Test public void multiPosTest() throws IOException { roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }"); + roundTripEquality("id:unittest:testmultipos::bamf", "{ \"geopos\": [ { \"lat\": -1.5, \"lng\": -1.5 }, { \"lat\": 63.4, \"lng\": 10.4 }, { \"lat\": 0.0, \"lng\": 0.0 } ] }"); } @Test diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 5ff9a4b0313..c457a703ecc 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; */ public class RPCNetwork implements Network, MethodHandler { - private static Logger log = Logger.getLogger(RPCNetwork.class.getName()); + private static final Logger log = Logger.getLogger(RPCNetwork.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final Identity identity; @@ -69,8 +69,8 @@ public class RPCNetwork implements Network, MethodHandler { private final LinkedHashMap<String, Route> lruRouteMap = new LinkedHashMap<>(10000, 0.5f, true); private final ExecutorService executor = new ThreadPoolExecutor(getNumThreads(), getNumThreads(), 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), - ThreadFactoryFactory.getDaemonThreadFactory("mbus.net"), new ThreadPoolExecutor.CallerRunsPolicy()); + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getDaemonThreadFactory("mbus.net")); private static int getNumThreads() { return Math.max(2, Runtime.getRuntime().availableProcessors()/2); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 3f1d290acf3..cdc6df1fdf2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -247,7 +247,17 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) if (putOp.getValidDbdId(_params._subDbId)) { if (putOp.changedDbdId() && useDocumentMetaStore(serialNum)) { - _gidToLidChangeHandler.notifyPut(token, docId.getGlobalId(), putOp.getLid(), serialNum); + /* + * Don't pass replay feed token to GidToLidChangeHandler. + * + * The passed feed token is kept until the ForceCommitDoneTask scheduled by the next + * force commit has completed. If a replay feed token containing an active throttler + * token is passed to GidToLidChangeHandler then + * TransactionLogReplayFeedHandler::make_replay_feed_token() might deadlock, waiting for + * active throttler tokens to be destroyed. + */ + FeedToken token_copy = (token && !token->is_replay()) ? token : FeedToken(); + _gidToLidChangeHandler.notifyPut(std::move(token_copy), docId.getGlobalId(), putOp.getLid(), serialNum); } auto onWriteDone = createPutDoneContext(std::move(token), {}, get_pending_lid_token(putOp), doc, putOp.getLid()); putSummary(serialNum, putOp.getLid(), doc, onWriteDone); diff --git a/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp index 6bedfb5013e..fde02dc8435 100644 --- a/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp +++ b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp @@ -60,28 +60,26 @@ void verify_sampling(size_t thread_id, size_t num_threads, std::vector<Sampler*> TEST_BARRIER(); // #1 auto t0 = steady_clock::now(); std::vector<duration> pre_usage = sample(samplers); - auto pre_total = cpu_usage::RUsage::sample(); + auto pre_total = cpu_usage::total_cpu_usage(); TEST_BARRIER(); // #2 TEST_BARRIER(); // #3 auto t1 = steady_clock::now(); std::vector<duration> post_usage = sample(samplers); - auto post_total = cpu_usage::RUsage::sample(); + auto post_total = cpu_usage::total_cpu_usage(); TEST_BARRIER(); // #4 double wall = to_s(t1 - t0); std::vector<double> load(4, 0.0); for (size_t i = 0; i < 4; ++i) { load[i] = to_s(post_usage[i] - pre_usage[i]) / wall; } - double user_load = to_s(post_total.user - pre_total.user) / wall; - double system_load = to_s(post_total.system - pre_total.system) / wall; - double total_load = to_s(post_total.total() - pre_total.total()) / wall; + double total_load = to_s(post_total - pre_total) / wall; EXPECT_GREATER(load[3], load[0]); // NB: cannot expect total_load to be greater than load[3] // here due to mock loads being 'as expected' while valgrind // will cut all loads in about half. EXPECT_GREATER(total_load, load[0]); - fprintf(stderr, "loads: { %.2f, %.2f, %.2f, %.2f }\n", load[0], load[1], load[2], load[3]); - fprintf(stderr, "total load: %.2f (user: %.2f, system: %.2f)\n", total_load, user_load, system_load); + fprintf(stderr, "loads: { %.3f, %.3f, %.3f, %.3f }\n", load[0], load[1], load[2], load[3]); + fprintf(stderr, "total load: %.3f\n", total_load); } else { int idx = (thread_id - 1); double target_load = double(thread_id - 1) / (num_threads - 2); @@ -114,9 +112,9 @@ TEST("measure thread CPU clock overhead") { fprintf(stderr, "approx overhead per sample (thread CPU clock): %f us\n", min_time_us); } -TEST("measure RUsage overhead") { +TEST("measure total cpu usage overhead") { duration d; - double min_time_us = BenchmarkTimer::benchmark([&d]() noexcept { d = cpu_usage::RUsage::sample().total(); }, budget) * 1000000.0; + double min_time_us = BenchmarkTimer::benchmark([&d]() noexcept { d = cpu_usage::total_cpu_usage(); }, budget) * 1000000.0; fprintf(stderr, "approx overhead per RUsage sample: %f us\n", min_time_us); } @@ -437,7 +435,7 @@ void do_sample_cpu_usage(const EndTime &end_time) { if (!body.empty()) { body.append(", "); } - body.append(fmt("%s: %.2f", CpuUsage::name_of(CpuUsage::Category(i)).c_str(), load[i])); + body.append(fmt("%s: %.3f", CpuUsage::name_of(CpuUsage::Category(i)).c_str(), load[i])); } fprintf(stderr, "CPU: %s\n", body.c_str()); } diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp index 345da66cc39..5609b0d8d09 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp @@ -46,13 +46,11 @@ public: } // <unnamed> -RUsage -RUsage::sample() noexcept -{ - rusage usage; - memset(&usage, 0, sizeof(usage)); - getrusage(RUSAGE_SELF, &usage); - return {from_timeval(usage.ru_utime), from_timeval(usage.ru_stime)}; +duration total_cpu_usage() noexcept { + timespec ts; + memset(&ts, 0, sizeof(ts)); + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + return from_timespec(ts); } ThreadSampler::UP create_thread_sampler(bool force_mock_impl, double expected_load) { @@ -224,7 +222,7 @@ CpuUsage::do_sample() my_sample.merge(_usage); _usage = my_sample; } - auto total = cpu_usage::RUsage::sample().total(); + auto total = cpu_usage::total_cpu_usage(); for (size_t i = 0; i < index_of(Category::OTHER); ++i) { total -= my_sample[i]; } diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.h b/vespalib/src/vespa/vespalib/util/cpu_usage.h index b0e9f60311b..3c30937151c 100644 --- a/vespalib/src/vespa/vespalib/util/cpu_usage.h +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.h @@ -13,15 +13,9 @@ namespace vespalib { namespace cpu_usage { /** - * Uses getrusage to sample the total amount of user and system cpu - * time used so far. + * Samples the total CPU usage of this process so far. **/ -struct RUsage { - duration user; - duration system; - duration total() const { return user + system; } - static RUsage sample() noexcept; -}; +duration total_cpu_usage() noexcept; /** * Samples the total CPU usage of the thread that created it. Note diff --git a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp index e91be68a671..dd790bcaa0a 100644 --- a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp @@ -244,7 +244,10 @@ DynamicOperationThrottler::DynamicOperationThrottler(const DynamicThrottleParams { } -DynamicOperationThrottler::~DynamicOperationThrottler() = default; +DynamicOperationThrottler::~DynamicOperationThrottler() +{ + assert(_pending_ops == 0u); +} bool DynamicOperationThrottler::has_spare_capacity_in_active_window() noexcept |