diff options
26 files changed, 316 insertions, 28 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java index 07f79f16334..d985089b2cb 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java @@ -93,9 +93,9 @@ public class RankingExpressionTypeResolver extends Processor { throw new IllegalArgumentException(profile + " is strict but is missing a query profile type " + "declaration of features " + context.queryFeaturesNotDeclared()); else - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + profile + - " are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + + profile + " are not declared " + + "and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java index 4b6a22fc81a..f2f2a82b97c 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java @@ -456,7 +456,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java index 68ebc5e86aa..5ddf5aaff28 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java @@ -8,16 +8,23 @@ import java.util.List; */ public class NodeRepoStats { + private final double totalCost; + private final double totalAllocatedCost; private final Load load; private final Load activeLoad; private final List<ApplicationStats> applicationStats; - public NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + public NodeRepoStats(double totalCost, double totalAllocatedCost, + Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + this.totalCost = totalCost; + this.totalAllocatedCost = totalAllocatedCost; this.load = load; this.activeLoad = activeLoad; this.applicationStats = List.copyOf(applicationStats); } + public double totalCost() { return totalCost; } + public double totalAllocatedCost() { return totalAllocatedCost; } public Load load() { return load; } public Load activeLoad() { return activeLoad; } public List<ApplicationStats> applicationStats() { return applicationStats; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java index 36677143f7a..75ac919f776 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java @@ -16,6 +16,12 @@ import java.util.stream.Collectors; @JsonInclude(JsonInclude.Include.NON_NULL) public class NodeRepoStatsData { + @JsonProperty("totalCost") + public Double totalCost; + + @JsonProperty("totalAllocatedCost") + public Double totalAllocatedCost; + @JsonProperty("load") public LoadData load; @@ -26,7 +32,8 @@ public class NodeRepoStatsData { public List<ApplicationStatsData> applications; public NodeRepoStats toNodeRepoStats() { - return new NodeRepoStats(load.toLoad(), activeLoad.toLoad(), + return new NodeRepoStats(totalCost, totalAllocatedCost, + load.toLoad(), activeLoad.toLoad(), applications.stream().map(stats -> stats.toApplicationStats()).collect(Collectors.toList())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java index 19f1ac5449f..96a3c9f177d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java @@ -30,6 +30,8 @@ public class StatsResponse extends SlimeJsonResponse { if (stats.applicationStats().isEmpty()) continue; // skip empty zones Cursor zoneObject = zonesArray.addObject(); zoneObject.setString("id", zone.toString()); + zoneObject.setDouble("totalCost", stats.totalCost()); + zoneObject.setDouble("totalAllocatedCost", stats.totalAllocatedCost()); toSlime(stats.load(), zoneObject.setObject("load")); toSlime(stats.activeLoad(), zoneObject.setObject("activeLoad")); Cursor applicationsArray = zoneObject.setArray("applications"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 3f1ca3f9706..93ebdcf3171 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -107,7 +107,7 @@ public class NodeRepositoryMock implements NodeRepository { .collect(Collectors.toList()) : List.of(); - return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); + return new NodeRepoStats(0.0, 0.0, Load.zero(), Load.zero(), applicationStats); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json index 673767c13a0..44b52e5be2c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json @@ -2,6 +2,8 @@ "zones": [ { "id": "prod.us-east-3", + "totalCost": 0.0, + "totalAllocatedCost": 0.0, "load": { "cpu": 0.0, "memory": 0.0, @@ -27,6 +29,8 @@ }, { "id": "prod.us-west-1", + "totalCost": 0.0, + "totalAllocatedCost": 0.0, "load": { "cpu": 0.0, "memory": 0.0, diff --git a/dist/vespa.spec b/dist/vespa.spec index de1b5382040..b94a77491b8 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -282,13 +282,14 @@ Requires: vespa-xxhash = 0.8.0 Requires: xxhash Requires: xxhash-libs >= 0.8.0 %endif -Requires: zlib -Requires: perf Requires: gdb +Requires: hostname +Requires: perf Requires: nc Requires: nghttp2 Requires: net-tools Requires: unzip +Requires: zlib Requires: zstd %if 0%{?el7} Requires: llvm7.0 diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 52dac0cfeea..5b26235b62e 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -66,10 +66,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <showDeprecation>true</showDeprecation> - </configuration> </plugin> </plugins> </build> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index 085b89d1253..1460ce70686 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -26,16 +26,23 @@ import java.util.Set; */ public class NodeRepoStats { + private final double totalCost; + private final double totalAllocatedCost; private final Load load; private final Load activeLoad; private final List<ApplicationStats> applicationStats; - private NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + private NodeRepoStats(double totalCost, double totalAllocatedCost, Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + this.totalCost = totalCost; + this.totalAllocatedCost = totalAllocatedCost; this.load = load; this.activeLoad = activeLoad; this.applicationStats = List.copyOf(applicationStats); } + public double totalCost() { return totalCost; } + public double totalAllocatedCost() { return totalAllocatedCost; } + /** * Returns the current average work-extracting utilization in this node repo over all nodes. * Capacity not allocated to active nodes are taken to have 0 utilization as it provides no useful work. @@ -50,11 +57,15 @@ public class NodeRepoStats { public static NodeRepoStats computeOver(NodeRepository nodeRepository) { NodeList allNodes = nodeRepository.nodes().list(); - List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of()); + double totalCost = allNodes.hosts().stream().mapToDouble(host -> host.resources().cost()).sum(); + double totalAllocatedCost = allNodes.not().hosts().stream() + .filter(node -> node.allocation().isPresent()) + .mapToDouble(node -> node.resources().cost()).sum(); + List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of()); Pair<Load, Load> load = computeLoad(allNodes, allNodeTimeseries); List<ApplicationStats> applicationStats = computeApplicationStats(allNodes, allNodeTimeseries); - return new NodeRepoStats(load.getFirst(), load.getSecond(), applicationStats); + return new NodeRepoStats(totalCost, totalAllocatedCost, load.getFirst(), load.getSecond(), applicationStats); } private static Pair<Load, Load> computeLoad(NodeList allNodes, List<NodeTimeseries> allNodeTimeseries) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 829823913a9..aa1abb18d8c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -131,7 +131,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { infrastructureProvisionInterval = Duration.ofMinutes(3); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); - nodeFailerInterval = Duration.ofMinutes(15); + nodeFailerInterval = Duration.ofMinutes(9); nodeFailureStatusUpdateInterval = Duration.ofMinutes(2); nodeMetricsCollectionInterval = Duration.ofMinutes(1); expeditedChangeRedeployInterval = Duration.ofMinutes(3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index c5d8b2518e5..dcfdb32e374 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -453,6 +453,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); + root.setDouble("total-cost", stats.totalCost()); + root.setDouble("total-allocated-cost", stats.totalAllocatedCost()); toSlime(stats.load(), root.setObject("load")); toSlime(stats.activeLoad(), root.setObject("activeLoad")); Cursor applicationsArray = root.setArray("applications"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java index 62c96af7629..788c56e08c6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java @@ -31,9 +31,13 @@ public class NodeRepoStatsTest { @Test public void testEmpty() { var tester = new NodeRepositoryTester(); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); - assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + var stats = tester.nodeRepository().computeStats(); + + assertEquals(0, stats.totalCost(), delta); + assertEquals(0, stats.totalAllocatedCost(), delta); + assertLoad(Load.zero(), stats.load()); + assertLoad(Load.zero(), stats.activeLoad()); + assertTrue(stats.applicationStats().isEmpty()); } @Test @@ -42,9 +46,13 @@ public class NodeRepoStatsTest { tester.addHost("host1", "default"); tester.addHost("host2", "default"); tester.addHost("host3", "small"); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); - assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + var stats = tester.nodeRepository().computeStats(); + + assertEquals(0.76, stats.totalCost(), delta); + assertEquals(0, stats.totalAllocatedCost(), delta); + assertLoad(Load.zero(), stats.load()); + assertLoad(Load.zero(), stats.activeLoad()); + assertTrue(stats.applicationStats().isEmpty()); } @Test @@ -97,6 +105,9 @@ public class NodeRepoStatsTest { var stats = tester.nodeRepository().computeStats(); + assertEquals(26, stats.totalCost(), delta); + assertEquals(8.319999999999999, stats.totalAllocatedCost(), delta); + assertLoad(new Load(0.6180,0.5562,0.4944), stats.load()); assertLoad(new Load(0.4682,0.4214,0.3745), stats.activeLoad()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json index 8a46f8115be..017a45d2bbe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json @@ -1,4 +1,6 @@ { + "total-cost" : 8.591999999999999, + "total-allocated-cost": 5.356, "load": { "cpu": 0.0, "memory": 0.0, diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 17b46499317..8c34f1b2016 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_define_module( src/tests/docsummary src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw + src/tests/docsummary/document_id_dfw src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/docsummary/summary_field_converter diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt new file mode 100644 index 00000000000..3591a36c8de --- /dev/null +++ b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_document_id_dfw_test_app TEST + SOURCES + document_id_dfw_test.cpp + DEPENDS + searchsummary + searchsummary_test + GTest::GTest +) +vespa_add_test(NAME searchsummary_document_id_dfw_test_app COMMAND searchsummary_document_id_dfw_test_app) diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp new file mode 100644 index 00000000000..0ea78b722c0 --- /dev/null +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -0,0 +1,153 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/documentid.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchsummary/docsummary/docsum_blob_entry_filter.h> +#include <vespa/searchsummary/docsummary/docsum_store_document.h> +#include <vespa/searchsummary/docsummary/document_id_dfw.h> +#include <vespa/searchsummary/docsummary/general_result.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> +#include <vespa/searchsummary/docsummary/resultpacker.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <iostream> +#include <memory> + +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::config_builder::DocumenttypesConfigBuilderHelper; +using document::config_builder::Struct; +using search::docsummary::DocsumBlobEntryFilter; +using search::docsummary::DocsumStoreDocument; +using search::docsummary::DocsumStoreValue; +using search::docsummary::DocumentIdDFW; +using search::docsummary::IDocsumStoreDocument; +using search::docsummary::ResultClass; +using search::docsummary::ResultConfig; +using search::docsummary::ResultPacker; +using search::docsummary::GeneralResult; +using vespalib::Slime; +using vespalib::slime::Cursor; +using vespalib::slime::ObjectInserter; +using vespalib::slime::SlimeInserter; + +namespace { + +const int32_t doc_type_id = 787121340; +const vespalib::string doc_type_name = "test"; +const vespalib::string header_name = doc_type_name + ".header"; +const vespalib::string body_name = doc_type_name + ".body"; + + +std::unique_ptr<const DocumentTypeRepo> +make_doc_type_repo() +{ + DocumenttypesConfigBuilderHelper builder; + builder.document(doc_type_id, doc_type_name, + Struct(header_name), Struct(body_name)); + return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); +} + +class DocumentIdDFWTest : public ::testing::Test +{ + vespalib::string _field_name; + vespalib::Memory _field_name_view; + DocsumBlobEntryFilter _docsum_blob_entry_filter; + std::unique_ptr<ResultConfig> _result_config; + std::unique_ptr<ResultPacker> _packer; // owns docsum blob + std::unique_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + +protected: + DocumentIdDFWTest(); + ~DocumentIdDFWTest() override; + + std::unique_ptr<DocsumStoreDocument> make_docsum_store_document(const vespalib::string &id); + std::unique_ptr<DocsumStoreValue> make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc); + vespalib::Slime write(const DocsumStoreValue& value); + vespalib::Memory get_field_name_view() const noexcept { return _field_name_view; } +}; + +DocumentIdDFWTest::DocumentIdDFWTest() + : testing::Test(), + _field_name("documentid"), + _field_name_view(_field_name.data(), _field_name.size()), + _docsum_blob_entry_filter(DocsumBlobEntryFilter().add_skip(search::docsummary::RES_LONG_STRING)), + _result_config(std::make_unique<ResultConfig>(_docsum_blob_entry_filter)), + _packer(std::make_unique<ResultPacker>(_result_config.get())), + _repo(make_doc_type_repo()), + _document_type(_repo->getDocumentType(doc_type_name)) +{ + auto* cfg = _result_config->AddResultClass("default", 0); + cfg->AddConfigEntry(_field_name.c_str(), search::docsummary::RES_LONG_STRING); + _result_config->CreateEnumMaps(); +} + + +DocumentIdDFWTest::~DocumentIdDFWTest() = default; + + +std::unique_ptr<DocsumStoreDocument> +DocumentIdDFWTest::make_docsum_store_document(const vespalib::string& id) +{ + auto doc = std::make_unique<Document>(*_document_type, DocumentId(id)); + doc->setRepo(*_repo); + return std::make_unique<DocsumStoreDocument>(std::move(doc)); +} + +std::unique_ptr<DocsumStoreValue> +DocumentIdDFWTest::make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc) +{ + EXPECT_TRUE(_packer->Init(0)); + const char *ptr = nullptr; + uint32_t len = 0; + EXPECT_TRUE(_packer->GetDocsumBlob(&ptr, &len)); + return std::make_unique<DocsumStoreValue>(ptr, len, std::move(doc)); +} + +vespalib::Slime +DocumentIdDFWTest::write(const DocsumStoreValue& value) +{ + auto result = std::make_unique<GeneralResult>(_result_config->LookupResultClass(0)); + EXPECT_TRUE(result->inplaceUnpack(value)); + Slime slime; + SlimeInserter top_inserter(slime); + Cursor & docsum = top_inserter.insertObject(); + ObjectInserter field_inserter(docsum, _field_name_view); + DocumentIdDFW writer; + writer.insertField(0, result.get(), nullptr, search::docsummary::RES_LONG_STRING, field_inserter); + return slime; +} + +TEST_F(DocumentIdDFWTest, insert_document_id) +{ + vespalib::string id("id::test::0"); + auto doc = make_docsum_store_document(id); + auto dsvalue = make_docsum_store_value(std::move(doc)); + auto slime = write(*dsvalue); + EXPECT_TRUE(slime.get()[get_field_name_view()].valid()); + EXPECT_EQ(id, slime.get()[get_field_name_view()].asString().make_string()); +} + +TEST_F(DocumentIdDFWTest, insert_document_id_no_document_doc) +{ + auto dsvalue = make_docsum_store_value(std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>())); + auto slime = write(*dsvalue); + EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); +} + +TEST_F(DocumentIdDFWTest, insert_document_id_no_docsum_store_doc) +{ + auto dsvalue = make_docsum_store_value({}); + auto slime = write(*dsvalue); + EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index e3272fb36de..e262482cddb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchsummary_docsummary OBJECT docsumstate.cpp docsumstorevalue.cpp docsumwriter.cpp + document_id_dfw.cpp dynamicteaserdfw.cpp empty_dfw.cpp general_result.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index e525989e972..e6c8d0b6ab8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -5,6 +5,7 @@ #include "summaryfieldconverter.h" #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/data/slime/inserter.h> namespace search::docsummary { @@ -39,4 +40,14 @@ DocsumStoreDocument::insert_summary_field(const vespalib::string& field_name, ve } } +void +DocsumStoreDocument::insert_document_id(vespalib::slime::Inserter& inserter) const +{ + if (_document) { + auto id = _document->getId().toString(); + vespalib::Memory id_view(id.data(), id.size()); + inserter.insertString(id_view); + } +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h index 66a5a74fa8d..3b0bea6e721 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h @@ -19,6 +19,7 @@ public: ~DocsumStoreDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 376d4f90204..fea11923858 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -4,6 +4,7 @@ #include "attribute_combiner_dfw.h" #include "copy_dfw.h" #include "docsumwriter.h" +#include "document_id_dfw.h" #include "empty_dfw.h" #include "geoposdfw.h" #include "idocsumenvironment.h" @@ -113,6 +114,9 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & *attr_ctx, matching_elems_fields); rc = static_cast<bool>(fieldWriter); } + } else if (overrideName == "documentid") { + fieldWriter = std::make_unique<DocumentIdDFW>(); + rc = true; } else { throw IllegalArgumentException("unknown override operation '" + overrideName + "' for field '" + fieldName + "'."); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp new file mode 100644 index 00000000000..8fcf7c95cdc --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_id_dfw.h" +#include "general_result.h" +#include "i_docsum_store_document.h" + +namespace search::docsummary { + +DocumentIdDFW::DocumentIdDFW() +{ +} + +DocumentIdDFW::~DocumentIdDFW() = default; + +bool +DocumentIdDFW::IsGenerated() const +{ + return false; +} + +void +DocumentIdDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *, ResType, + vespalib::slime::Inserter &target) +{ + const auto* document = gres->get_document(); + if (document != nullptr) { + document->insert_document_id(target); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h new file mode 100644 index 00000000000..f6353912384 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +/* + * Class for writing document id field. + */ +class DocumentIdDFW : public DocsumFieldWriter +{ +private: +public: + DocumentIdDFW(); + ~DocumentIdDFW() override; + bool IsGenerated() const override; + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h index f81412eb34c..c177568c467 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h @@ -22,6 +22,7 @@ public: virtual ~IDocsumStoreDocument() = default; virtual std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const = 0; virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const = 0; + virtual void insert_document_id(vespalib::slime::Inserter& inserter) const = 0; }; } diff --git a/security-utils/pom.xml b/security-utils/pom.xml index d4b4bc8d0f6..0a2e1251d6a 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -71,10 +71,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <showDeprecation>true</showDeprecation> - </configuration> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -89,7 +85,10 @@ </executions> </plugin> <plugin> - <!-- Build with maven-bundle-plugin to avoid depending on jdisc_core to get the correct Import-Packages --> + <!-- + Build with maven-bundle-plugin to avoid depending on jdisc_core to get the correct Import-Packages + Jackson and BC compile scope dependencies are not embedded. Packages are instead imported through manifest. + --> <groupId>org.apache.felix</groupId> <artifactId>maven-bundle-plugin</artifactId> <extensions>true</extensions> diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 20083b9160e..5bcead79f97 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -135,6 +135,7 @@ public: ~DocsumStoreVsmDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; DocsumStoreVsmDocument::DocsumStoreVsmDocument(const document::Document* document) @@ -168,6 +169,16 @@ DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, } } +void +DocsumStoreVsmDocument::insert_document_id(vespalib::slime::Inserter& inserter) const +{ + if (_document) { + auto id = _document->getId().toString(); + vespalib::Memory id_view(id.data(), id.size()); + inserter.insertString(id_view); + } +} + } FieldPath |