diff options
22 files changed, 196 insertions, 59 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index c932958b58b..0b36ea3bee2 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -303,6 +303,8 @@ "public static java.lang.String toMessageString(java.lang.Throwable)", "public java.util.Optional athenzDomain()", "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public boolean equals(java.lang.Object)", + "public int hashCode()", "public static void main(java.lang.String[])" ], "fields": [ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index a6cecefe940..d15d76fd11b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -267,6 +267,27 @@ public class DeploymentSpec { return Optional.ofNullable(athenzService); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeploymentSpec that = (DeploymentSpec) o; + return globalServiceId.equals(that.globalServiceId) && + upgradePolicy == that.upgradePolicy && + majorVersion.equals(that.majorVersion) && + changeBlockers.equals(that.changeBlockers) && + steps.equals(that.steps) && + xmlForm.equals(that.xmlForm) && + athenzDomain.equals(that.athenzDomain) && + athenzService.equals(that.athenzService) && + notifications.equals(that.notifications); + } + + @Override + public int hashCode() { + return Objects.hash(globalServiceId, upgradePolicy, majorVersion, changeBlockers, steps, xmlForm, athenzDomain, athenzService, notifications); + } + /** This may be invoked by a continuous build */ public static void main(String[] args) { if (args.length != 2 && args.length != 3) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java index f9469b1ae65..3c24cb58cff 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -44,7 +44,7 @@ public class UrlDownloadRpcServer { private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; private final File downloadBaseDir; - private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), + private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("Rpc download executor")); UrlDownloadRpcServer(Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index f240129eda1..991cd99968d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -212,6 +212,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { + log.log(LogLevel.INFO, "Redeployment of " + applicationsNotRedeployed + + " failed, will retry in " + sleepTimeWhenRedeployingFails); Thread.sleep(sleepTimeWhenRedeployingFails.toMillis()); } } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 16aaef048b5..3034c7dfd53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -85,8 +85,8 @@ public class FileServer { private FileServer(ConnectionPool connectionPool, File rootDir) { this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.pushExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - this.pullExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); + this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); } boolean hasFile(String fileReference) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 1ce90fad465..e8abecc3236 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -126,7 +126,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { this.metrics = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.hostLivenessTracker = hostLivenessTracker; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); - int numberOfRpcThreads = (config.numRpcThreads() == 0) ? Runtime.getRuntime().availableProcessors() : config.numRpcThreads(); + int numberOfRpcThreads = (config.numRpcThreads() == 0) + ? Math.max(8, Runtime.getRuntime().availableProcessors()) + : config.numRpcThreads(); executorService = new ThreadPoolExecutor(numberOfRpcThreads, numberOfRpcThreads, 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index 612941ece7a..5700e316493 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; /** @@ -69,7 +70,7 @@ public class FS4InvokerFactory { * @return Optional containing the SearchInvoker or <i>empty</i> if some node in the * list is invalid and the remaining coverage is not sufficient */ - public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { + public Optional<SearchInvoker> getSearchInvoker(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 1d39cffa9d2..146b132be22 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; /** @@ -106,7 +107,7 @@ public class Dispatcher extends AbstractComponent { @FunctionalInterface private interface SearchInvokerSupplier { - Optional<SearchInvoker> supply(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage); + Optional<SearchInvoker> supply(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage); } // build invoker based on searchpath @@ -121,7 +122,7 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } else { query.trace(false, 2, "Dispatching internally with search path ", searchPath); - return invokerFactory.supply(query, -1, nodes, true); + return invokerFactory.supply(query, OptionalInt.empty(), nodes, true); } } catch (InvalidSearchPathException e) { return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage()))); @@ -133,7 +134,7 @@ public class Dispatcher extends AbstractComponent { if (directNode.isPresent()) { Node node = directNode.get(); query.trace(false, 2, "Dispatching directly to ", node); - return invokerFactory.supply(query, -1, Arrays.asList(node), true); + return invokerFactory.supply(query, OptionalInt.empty(), Arrays.asList(node), true); } int covered = searchCluster.groupsWithSufficientCoverage(); @@ -148,7 +149,8 @@ public class Dispatcher extends AbstractComponent { } Group group = groupInCluster.get(); boolean acceptIncompleteCoverage = (i == max - 1); - Optional<SearchInvoker> invoker = invokerFactory.supply(query, group.id(), group.nodes(), acceptIncompleteCoverage); + Optional<SearchInvoker> invoker = invokerFactory.supply(query, OptionalInt.of(group.id()), group.nodes(), + acceptIncompleteCoverage); if (invoker.isPresent()) { query.trace(false, 2, "Dispatching internally to search group ", group.id()); query.getModel().setSearchPath("/" + group.id()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index e497ef6751b..a1658684b87 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -18,6 +18,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; @@ -62,11 +63,11 @@ public class SearchCluster implements NodeManager<Node> { public SearchCluster(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; - this.size = dispatchConfig.node().size(); this.fs4ResourcePool = fs4ResourcePool; this.vipStatus = vipStatus; List<Node> nodes = toNodes(dispatchConfig); + this.size = nodes.size(); // Create groups ImmutableMap.Builder<Integer, Group> groupsBuilder = new ImmutableMap.Builder<>(); @@ -331,9 +332,10 @@ public class SearchCluster implements NodeManager<Node> { } } - private void logIfInsufficientCoverage(boolean sufficient, int groupId, int nodes) { + private void logIfInsufficientCoverage(boolean sufficient, OptionalInt groupId, int nodes) { if (!sufficient) { - log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", groupId, nodes, groupSize(), + String group = groupId.isPresent()? Integer.toString(groupId.getAsInt()) : "(unspecified)"; + log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", group, nodes, groupSize(), groupSize() - dispatchConfig.maxNodesDownPerGroup())); } } @@ -341,14 +343,22 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - public boolean isPartialGroupCoverageSufficient(int groupId, List<Node> nodes) { + public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { if (orderedGroups.size() == 1) { boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); - logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + logIfInsufficientCoverage(sufficient, knownGroupId, nodes.size()); return sufficient; } - int nodesInGroup = groups.get(groupId).nodes().size(); + if (knownGroupId.isEmpty()) { + return false; + } + int groupId = knownGroupId.getAsInt(); + Group group = groups.get(groupId); + if (group == null) { + return false; + } + int nodesInGroup = group.nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; for (Group g : orderedGroups) { @@ -363,7 +373,7 @@ public class SearchCluster implements NodeManager<Node> { } long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; boolean sufficient = isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); - logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + logIfInsufficientCoverage(sufficient, knownGroupId, nodes.size()); return sufficient; } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index fae659b0d70..708caafa3f5 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.util.List; import java.util.Optional; +import java.util.OptionalInt; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.hamcrest.Matchers.is; @@ -112,7 +113,7 @@ public class DispatcherTest { } @Override - public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { + public Optional<SearchInvoker> getSearchInvoker(Query query, OptionalInt groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { if (step >= events.length) { throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 3efb0db8cce..7316a859382 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -270,20 +270,20 @@ public class ApplicationController { .map(app -> new LockedApplication(app, lock)) .orElseGet(() -> new LockedApplication(createApplication(applicationId, Optional.empty()), lock)); - boolean canDeployDirectly = options.deployDirectly || zone.environment().isManuallyDeployed(); + boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed(); boolean preferOldestVersion = options.deployCurrentVersion; // Determine versions to use. Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; - if (canDeployDirectly) { - platformVersion = options.vespaVersion.map(Version::new).orElse(application.get().deploymentSpec().majorVersion() - .flatMap(this::lastCompatibleVersion) - .orElse(controller.systemVersion())); + if (manuallyDeployed) { applicationVersion = applicationVersionFromDeployer.orElse(ApplicationVersion.unknown); applicationPackage = applicationPackageFromDeployer.orElseThrow( () -> new IllegalArgumentException("Application package must be given when deploying to " + zone)); + platformVersion = options.vespaVersion.map(Version::new).orElse(applicationPackage.deploymentSpec().majorVersion() + .flatMap(this::lastCompatibleVersion) + .orElse(controller.systemVersion())); } else { JobType jobType = JobType.from(controller.system(), zone) @@ -309,8 +309,9 @@ public class ApplicationController { verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); // Update application with information from application package - if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally()) - // TODO jvenstad: Store only on submissions (not on deployments to dev!!) + if ( ! preferOldestVersion && + ! application.get().deploymentJobs().deployedInternally() && + ! zone.environment().isManuallyDeployed()) application = storeWithUpdatedConfig(application, applicationPackage); // Assign global rotation diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 06417da8157..32068b006f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; @@ -426,7 +427,7 @@ public class ControllerTest { } @Test - public void testDeployDirectly() { + public void testIntegrationTestDeployment() { DeploymentTester tester = new DeploymentTester(); Version six = Version.fromString("6.1"); tester.upgradeSystem(six); @@ -461,6 +462,28 @@ public class ControllerTest { } @Test + public void testDevDeployment() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.dev) + .majorVersion(6) + .region("us-east-1") + .build(); + + // Create application + Application app = tester.createApplication("app1", "tenant1", 1, 2L); + ZoneId zone = ZoneId.from("dev", "us-east-1"); + + // Deploy + tester.controller().applications().deploy(app.id(), zone, Optional.of(applicationPackage), DeployOptions.none()); + assertTrue("Application deployed and activated", + tester.controllerTester().configServer().application(app.id()).get().activated()); + assertTrue("No job status added", + tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); + assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, tester.applications().require(app.id()).deploymentSpec()); + } + + @Test public void testSuspension() { DeploymentTester tester = new DeploymentTester(); Application app = tester.createApplication("app1", "tenant1", 1, 11L); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 443a2c23b6a..3744e44152a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -66,7 +66,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastCompleted": { @@ -80,7 +80,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastSuccess": { @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" } }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 64c92f9cde4..822bc447d8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -66,7 +66,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastCompleted": { @@ -80,7 +80,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" }, "lastSuccess": { @@ -94,7 +94,7 @@ "gitCommit": "commit1" } }, - "reason": "Testing deployment for production-us-central-1 (platform 6.1, application 1.0.42-commit1)", + "reason": "Testing last changes outside prod", "at": "(ignore)" } }, diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 1053c5ff44d..418837ca2a0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -152,8 +152,11 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeWeightedSet(JsonGenerator generator, FieldBase field, WeightedSet<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); - wrapIOException(() -> { generator.writeStartObject(); @@ -168,8 +171,11 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeCollectionField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, CollectionFieldValue<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); - wrapIOException(() -> { generator.writeStartArray(); Iterator<T> i = value.iterator(); @@ -184,6 +190,10 @@ public class JsonSerializationHelper { public static <K extends FieldValue, V extends FieldValue> void serializeMapField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, MapFieldValue<K, V> map) { + // Hide empty fields + if (map.size() == 0) { + return; + } fieldNameIfNotNull(generator, field); wrapIOException(() -> { generator.writeStartObject(); @@ -203,6 +213,10 @@ public class JsonSerializationHelper { } public static <T extends FieldValue> void serializeArrayField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Array<T> value) { + // Hide empty fields + if (value.size() == 0) { + return; + } wrapIOException(() -> { fieldNameIfNotNull(generator, field); generator.writeStartArray(); diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index c7660e5d527..32d23349dc0 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -178,6 +178,12 @@ std::unique_ptr<Tensor> createExpectedUpdatedTensorWith2Cells() { {{{"x", "9"}, {"y", "9"}}, 11} }, {"x", "y"}); } +std::unique_ptr<Tensor> createExpectedAddUpdatedTensorWith3Cells() { + return createTensor({ {{{"x", "8"}, {"y", "8"}}, 2}, + {{{"x", "8"}, {"y", "9"}}, 2}, + {{{"x", "9"}, {"y", "9"}}, 11} }, {"x", "y"}); +} + FieldValue::UP createTensorFieldValueWith2Cells() { auto fv(std::make_unique<TensorFieldValue>()); *fv = createTensorWith2Cells(); @@ -973,14 +979,21 @@ DocumentUpdateTest::testTensorAddUpdate() updated.setValue(updated.getField("tensor"), *oldTensor); CPPUNIT_ASSERT(*doc != updated); testValueUpdate(*createTensorAddUpdate(), *DataType::TENSOR); + std::string expTensorAddUpdateString("TensorAddUpdate(" + "{TensorFieldValue: " + "{\"dimensions\":[\"x\",\"y\"]," + "\"cells\":[" + "{\"address\":{\"x\":\"8\",\"y\":\"9\"},\"value\":2}," + "{\"address\":{\"x\":\"8\",\"y\":\"8\"},\"value\":2}" + "]}})"); + CPPUNIT_ASSERT_EQUAL(expTensorAddUpdateString, createTensorAddUpdate()->toString()); DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(*createTensorAddUpdate())); upd.applyTo(updated); FieldValue::UP fval(updated.getValue("tensor")); CPPUNIT_ASSERT(fval); auto &tensor = asTensor(*fval); - // Note: Placeholder value for now - auto expectedUpdatedTensor = createTensorWith2Cells(); + auto expectedUpdatedTensor = createExpectedAddUpdatedTensorWith3Cells(); CPPUNIT_ASSERT(tensor.equals(*expectedUpdatedTensor)); } @@ -994,6 +1007,13 @@ DocumentUpdateTest::testTensorModifyUpdate() updated.setValue(updated.getField("tensor"), *oldTensor); CPPUNIT_ASSERT(*doc != updated); testValueUpdate(*createTensorModifyUpdate(), *DataType::TENSOR); + std::string expTensorModifyUpdateString("TensorModifyUpdate(replace," + "{TensorFieldValue: " + "{\"dimensions\":[\"x\",\"y\"]," + "\"cells\":[" + "{\"address\":{\"x\":\"8\",\"y\":\"9\"},\"value\":2}" + "]}})"); + CPPUNIT_ASSERT_EQUAL(expTensorModifyUpdateString, createTensorModifyUpdate()->toString()); DocumentUpdate upd(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); upd.addUpdate(FieldUpdate(upd.getType().getField("tensor")).addUpdate(*createTensorModifyUpdate())); upd.applyTo(updated); diff --git a/document/src/vespa/document/update/tensoraddupdate.cpp b/document/src/vespa/document/update/tensoraddupdate.cpp index eb708d9f651..714eee28ff3 100644 --- a/document/src/vespa/document/update/tensoraddupdate.cpp +++ b/document/src/vespa/document/update/tensoraddupdate.cpp @@ -81,7 +81,11 @@ TensorAddUpdate::checkCompatibility(const Field& field) const std::unique_ptr<Tensor> TensorAddUpdate::applyTo(const Tensor &tensor) const { - return tensor.clone(); + auto &addTensor = _tensor->getAsTensorPtr(); + if (addTensor) { + return tensor.add(*addTensor); + } + return std::unique_ptr<Tensor>(); } bool @@ -112,9 +116,11 @@ TensorAddUpdate::printXml(XmlOutputStream& xos) const void TensorAddUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { - (void) verbose; - (void) indent; - out << "{TensorAddUpdate::print not yet implemented}"; + out << indent << "TensorAddUpdate("; + if (_tensor) { + _tensor->print(out, verbose, indent); + } + out << ")"; } void diff --git a/document/src/vespa/document/update/tensormodifyupdate.cpp b/document/src/vespa/document/update/tensormodifyupdate.cpp index a02379e4991..8cee367cae0 100644 --- a/document/src/vespa/document/update/tensormodifyupdate.cpp +++ b/document/src/vespa/document/update/tensormodifyupdate.cpp @@ -50,6 +50,23 @@ getJoinFunction(TensorModifyUpdate::Operation operation) } } +vespalib::string +getJoinFunctionName(TensorModifyUpdate::Operation operation) +{ + using Operation = TensorModifyUpdate::Operation; + + switch (operation) { + case Operation::REPLACE: + return "replace"; + case Operation::ADD: + return "add"; + case Operation::MUL: + return "multiply"; + default: + throw IllegalArgumentException("Bad operation", VESPA_STRLOC); + } +} + } IMPLEMENT_IDENTIFIABLE(TensorModifyUpdate, ValueUpdate); @@ -156,9 +173,11 @@ TensorModifyUpdate::printXml(XmlOutputStream& xos) const void TensorModifyUpdate::print(std::ostream& out, bool verbose, const std::string& indent) const { - (void) verbose; - (void) indent; - out << "{TensorModifyUpdate::print not yet implemented}"; + out << indent << "TensorModifyUpdate(" << getJoinFunctionName(_operation) << ","; + if (_tensor) { + _tensor->print(out, verbose, indent); + } + out << ")"; } void diff --git a/node-admin/pom.xml b/node-admin/pom.xml index fda4acebfa0..d550ceb7c9d 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -55,6 +55,16 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <scope>provided</scope> + </dependency> <!-- Compile --> <dependency> diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 40c2733d230..fb8674b5255 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -778,12 +778,7 @@ Test::requireThatAttributesAreUsed() "bj", *rep, 0, rclass)); // empty doc - EXPECT_TRUE(assertSlime("{bd:[]," - "be:[]," - "bf:[]," - "bg:[]," - "bh:[]," - "bi:[]}", *rep, 1, false)); + EXPECT_TRUE(assertSlime("{}", *rep, 1, false)); TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass)); proton::IAttributeManager::SP attributeManager = dc._ddb->getReadySubDB()->getAttributeManager(); @@ -807,9 +802,7 @@ Test::requireThatAttributesAreUsed() req3.hits.push_back(DocsumRequest::Hit(gid3)); DocsumReply::UP rep3 = dc._ddb->getDocsums(req3); - EXPECT_TRUE(assertSlime("{bd:[],be:[],bf:[],bg:[]," - "bh:[],bi:[]," - "bj:'0x01020178017901016101624010000000000000'}", + EXPECT_TRUE(assertSlime("{bj:'0x01020178017901016101624010000000000000'}", *rep3, 0, true)); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 19dd096c46c..1ca1a336d2d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -149,9 +149,13 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inser using vespalib::slime::Cursor; using vespalib::Memory; const IAttributeVector & v = vec(*state); - uint32_t entries = v.getValueCount(docid); bool isWeightedSet = v.hasWeightedSetType(); + uint32_t entries = v.getValueCount(docid); + if (entries == 0) { + return; // Don't insert empty fields + } + Cursor &arr = target.insertArray(); BasicType::Type t = v.getBasicType(); switch (t) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 91a7fd45061..72cedb05f7c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -231,10 +231,6 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor FieldValue::UP _field_value; FieldValueConverter &_structuredFieldConverter; - void visit(const ArrayFieldValue &value) override { - _field_value = _structuredFieldConverter.convert(value); - } - template <typename T> void visitPrimitive(const T &t) { _field_value.reset(t.clone()); @@ -274,8 +270,16 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor visitPrimitive(value); } - void visit(const MapFieldValue & v) override { - _field_value = _structuredFieldConverter.convert(v); + void visit(const ArrayFieldValue &value) override { + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string + } + + void visit(const MapFieldValue & value) override { + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string } void visit(const StructFieldValue &value) override { @@ -292,7 +296,9 @@ class SummaryFieldValueConverter : protected ConstFieldValueVisitor } void visit(const WeightedSetFieldValue &value) override { - _field_value = _structuredFieldConverter.convert(value); + if (value.size() > 0) { + _field_value = _structuredFieldConverter.convert(value); + } // else: implicit empty string } void visit(const TensorFieldValue &value) override { |