diff options
98 files changed, 656 insertions, 669 deletions
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 69e1a94a813..7971f4eebcf 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -45,7 +45,7 @@ <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>11.0.14</jetty.version> + <jetty.version>11.0.15</jetty.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20230227</org.json.version> <!-- TODO: Remove on Vespa 9 --> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index e5aa47fe5c9..b37fe02226b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -130,7 +130,7 @@ public class FileReceiver { moveFileToDestination(inprogressFile, file); } else { decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); - log.log(Level.FINE, () -> "compression type to use=" + compressionType); + log.log(Level.FINEST, () -> "compression type to use=" + compressionType); new FileReferenceCompressor(fileType, compressionType).decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } @@ -230,7 +230,7 @@ public class FileReceiver { } private void receiveFileMeta(Request req) { - log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINEST, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); String fileName = req.parameters().get(1).asString(); Type type = FileReferenceData.Type.valueOf(req.parameters().get(2).asString()); @@ -281,7 +281,7 @@ public class FileReceiver { } private void receiveFileEof(Request req) { - log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINEST, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); long xxhash = req.parameters().get(2).asInt64(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index 8d6f9ea1af3..5ab1841486e 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -61,7 +61,7 @@ public class FileReferenceCompressor { } public void decompress(File inputFile, File outputDir) throws IOException { - log.log(Level.FINE, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + log.log(Level.FINEST, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); try (ArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { @@ -121,7 +121,7 @@ public class FileReferenceCompressor { private OutputStream compressedOutputStream(File outputFile) throws IOException { switch (type) { case compressed: - log.log(Level.FINE, () -> "Compressing with compression type " + compressionType); + log.log(Level.FINEST, () -> "Compressing with compression type " + compressionType); return switch (compressionType) { case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile)); case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile)); @@ -137,7 +137,7 @@ public class FileReferenceCompressor { private InputStream decompressedInputStream(File inputFile) throws IOException { switch (type) { case compressed: - log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType); + log.log(Level.FINEST, () -> "Decompressing with compression type " + compressionType); return switch (compressionType) { case gzip -> new GZIPInputStream(new FileInputStream(inputFile)); case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile)); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e89724dfb16..0a5cd1aacac 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -410,12 +410,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID); - public static final UnboundBooleanFlag NEW_IDDOC_LAYOUT = defineFeatureFlag( - "new_iddoc_layout", false, List.of("tokle", "bjorncs", "olaa"), "2023-04-24", "2023-05-31", - "Whether to use new identity document lauoyt", - "Takes effect on node reboot", - HOSTNAME); - /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index c9c76e1edd3..3fb9c73367d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClientException; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator; @@ -77,7 +76,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; private final BooleanFlag tenantServiceIdentityFlag; - private final BooleanFlag useNewIdentityDocumentLayout; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -99,7 +97,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.clock = clock; this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); - this.useNewIdentityDocumentLayout = Flags.NEW_IDDOC_LAYOUT.bindTo(flagSource); } public boolean converge(NodeAgentContext context) { @@ -133,7 +130,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Instant now = clock.instant(); Instant expiry = certificate.getNotAfter().toInstant(); var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); - if (refreshIdentityDocument(doc, context)) { + if (doc.outdated()) { context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion()); registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); return true; @@ -153,7 +150,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return false; } else { lastRefreshAttempt.put(context.containerName(), now); - refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc.identityDocument(), identityType, athenzIdentity); + refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc, identityType, athenzIdentity); return true; } } @@ -164,11 +161,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } - private boolean refreshIdentityDocument(SignedIdentityDocument signedIdentityDocument, NodeAgentContext context) { - int expectedVersion = documentVersion(context); - return signedIdentityDocument.outdated() || signedIdentityDocument.documentVersion() != expectedVersion; - } - public void clearCredentials(NodeAgentContext context) { FileFinder.files(context.paths().of(CONTAINER_SIA_DIRECTORY)) .deleteRecursively(context); @@ -208,8 +200,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - SignedIdentityDocument signedDoc = signedIdentityDocument(context, identityType); - IdentityDocument doc = signedDoc.identityDocument(); + SignedIdentityDocument doc = signedIdentityDocument(context, identityType); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); @@ -221,9 +212,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ztsClient.registerInstance( doc.providerService(), identity, - EntityBindingsMapper.toAttestationData(signedDoc), + EntityBindingsMapper.toAttestationData(doc), csr); - EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, signedDoc); + EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, doc); writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate()); context.log(logger, "Instance successfully registered and credentials written to file"); } @@ -232,14 +223,14 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { /** * Return zts url from identity document, fallback to ztsEndpoint */ - private URI ztsEndpoint(IdentityDocument doc) { + private URI ztsEndpoint(SignedIdentityDocument doc) { return Optional.ofNullable(doc.ztsUrl()) .filter(s -> !s.isBlank()) .map(URI::create) .orElse(ztsEndpoint); } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, - ContainerPath identityDocumentFile, IdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { + ContainerPath identityDocumentFile, SignedIdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( @@ -300,8 +291,8 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) { return switch (identityType) { - case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value(), documentVersion(context)); - case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)); + case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value()); }; } @@ -314,9 +305,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private AthenzIdentity getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) { if (Files.exists(identityDocumentFile)) { - return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).identityDocument().serviceIdentity(); + return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).serviceIdentity(); } else { - return identityDocumentClient.getTenantIdentityDocument(context.hostname().value(), documentVersion(context)).identityDocument().serviceIdentity(); + return identityDocumentClient.getTenantIdentityDocument(context.hostname().value()).serviceIdentity(); } } @@ -326,17 +317,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .value(); } - /* - Get the document version to ask for - */ - private int documentVersion(NodeAgentContext context) { - return useNewIdentityDocumentLayout - .with(FetchVector.Dimension.HOSTNAME, context.hostname().value()) - .value() - ? SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION - : SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; - } - enum IdentityType { NODE("vespa-node-identity-document.json"), TENANT("vespa-tenant-identity-document.json"); diff --git a/parent/pom.xml b/parent/pom.xml index e6ef7e873fd..f3b9fc9baeb 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1155,7 +1155,7 @@ <felix.log.version>1.0.1</felix.log.version> <findbugs.version>3.0.2</findbugs.version> <!-- Should be kept in sync with guava --> <hdrhistogram.version>2.1.12</hdrhistogram.version> - <jetty.version>11.0.14</jetty.version> + <jetty.version>11.0.15</jetty.version> <jetty-servlet-api.version>5.0.2</jetty-servlet-api.version> <jjwt.version>0.11.2</jjwt.version> <jna.version>5.11.0</jna.version> diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index fd6f6af730c..4ad386afa3f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -16,7 +16,6 @@ #include <vespa/searchlib/aggregation/grouping.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/attribute/extendableattributes.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/engine/docsumreply.h> #include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchreply.h> @@ -36,6 +35,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -64,6 +64,7 @@ using search::index::schema::DataType; using storage::spi::Timestamp; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; +using vespalib::FeatureSet; using vespalib::nbostream; vespalib::ThreadBundle &ttb() { return vespalib::ThreadBundle::trivial(); } diff --git a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp index 2027ad56768..fe7303692ba 100644 --- a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp +++ b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp @@ -20,7 +20,6 @@ ProtonConfig make_proton_config(double concurrency, uint32_t indexing_threads = 1) { ProtonConfigBuilder builder; - // This setup requires a minimum of 4 shared threads. builder.documentdb.push_back(ProtonConfig::Documentdb()); builder.documentdb.push_back(ProtonConfig::Documentdb()); builder.flush.maxconcurrent = 1; @@ -48,8 +47,10 @@ expect_field_writer_threads(uint32_t exp_threads, uint32_t cpu_cores, uint32_t i TEST(SharedThreadingServiceConfigTest, shared_threads_are_derived_from_cpu_cores_and_feeding_concurrency) { - expect_shared_threads(4, 1); - expect_shared_threads(4, 6); + expect_shared_threads(2, 1); + expect_shared_threads(2, 4); + expect_shared_threads(3, 5); + expect_shared_threads(3, 6); expect_shared_threads(4, 8); expect_shared_threads(5, 9); expect_shared_threads(5, 10); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index 82e1aa3b57c..430412fc1c0 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -137,8 +137,15 @@ SearchContext::getStore() const SearchContext::SearchContext(QueryTermSimple::UP qTerm, const DocumentMetaStore &toBeSearched) : search::attribute::SearchContext(toBeSearched), - _isWord(qTerm->isWord()) + _isWord(qTerm->isWord()), + _docid_limit(toBeSearched.getCommittedDocIdLimit()) { } +uint32_t +SearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h index ca4b026e2a4..7c88d8f3502 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h @@ -18,6 +18,7 @@ private: bool _isWord; document::GlobalId _gid; + uint32_t _docid_limit; unsigned int approximateHits() const override; int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; @@ -30,6 +31,7 @@ private: public: SearchContext(std::unique_ptr<search::QueryTermSimple> qTerm, const DocumentMetaStore &toBeSearched); + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index fd5c3782b9a..6014df1c2f9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -18,7 +18,6 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.docsum_matcher"); -using search::FeatureSet; using search::MatchingElements; using search::MatchingElementsFields; using search::fef::FeatureResolver; @@ -29,6 +28,7 @@ using search::queryeval::IntermediateBlueprint; using search::queryeval::MatchingElementsSearch; using search::queryeval::SameElementBlueprint; using search::queryeval::SearchIterator; +using vespalib::FeatureSet; using AttrSearchCtx = search::attribute::ISearchContext; diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h index 006a443e539..bf99a6b1950 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h @@ -2,9 +2,9 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> #include <memory> @@ -21,7 +21,7 @@ class SearchSession; class DocsumMatcher { private: - using FeatureSet = search::FeatureSet; + using FeatureSet = vespalib::FeatureSet; using MatchingElementsFields = search::MatchingElementsFields; using MatchingElements = search::MatchingElements; diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp index 8f7970f5717..30958214b72 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.cpp @@ -12,10 +12,10 @@ #include <vespa/searchlib/queryeval/searchiterator.h> using vespalib::Doom; +using vespalib::FeatureSet; +using vespalib::FeatureValues; using vespalib::Runnable; using vespalib::ThreadBundle; -using search::FeatureSet; -using search::FeatureValues; using search::fef::FeatureResolver; using search::fef::RankProgram; using search::queryeval::SearchIterator; diff --git a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h index 48c3476f164..09da89250a2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/extract_features.h +++ b/searchcore/src/vespa/searchcore/proton/matching/extract_features.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/stringmap.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> namespace vespalib { class Doom; }; @@ -16,8 +16,8 @@ namespace proton::matching { class MatchToolsFactory; struct ExtractFeatures { - using FeatureSet = search::FeatureSet; - using FeatureValues = search::FeatureValues; + using FeatureSet = vespalib::FeatureSet; + using FeatureValues = vespalib::FeatureValues; using ThreadBundle = vespalib::ThreadBundle; using SearchIterator = search::queryeval::SearchIterator; using RankProgram = search::fef::RankProgram; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 3a43e9a118e..0bb183d1dc0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -18,7 +18,7 @@ namespace proton::matching { using namespace search::fef; using search::queryeval::SearchIterator; -using search::FeatureSet; +using vespalib::FeatureSet; using vespalib::ThreadBundle; using vespalib::Issue; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 236964c2e6b..b393558638d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -27,7 +27,6 @@ using namespace search::engine; using namespace search::grouping; using search::DocumentMetaData; using search::LidUsageStats; -using search::FeatureSet; using search::MatchingElementsFields; using search::MatchingElements; using search::attribute::IAttributeContext; @@ -39,6 +38,7 @@ using search::fef::indexproperties::hitcollector::ArraySize; using search::queryeval::Blueprint; using search::queryeval::SearchIterator; using vespalib::Doom; +using vespalib::FeatureSet; using vespalib::make_string_short::fmt; namespace proton::matching { diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index bad56fe1c36..6507ffca2eb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -11,13 +11,13 @@ #include "viewresolver.h" #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/fef/blueprintfactory.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/query/base.h> #include <vespa/vespalib/util/clock.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/thread_bundle.h> #include <mutex> @@ -135,7 +135,7 @@ public: * @param attrCtx abstract view of attribute data * @return calculated summary features. **/ - search::FeatureSet::SP + vespalib::FeatureSet::SP getSummaryFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionManager) const; @@ -149,7 +149,7 @@ public: * @param attrCtx abstract view of attribute data * @return calculated rank features. **/ - search::FeatureSet::SP + vespalib::FeatureSet::SP getRankFeatures(const DocsumRequest & req, ISearchContext & searchCtx, IAttributeContext & attrCtx, SessionManager &sessionManager) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp index 50ef5039e75..c1802b40deb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -28,10 +28,10 @@ namespace { uint32_t derive_shared_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) { - uint32_t scaled_cores = (uint32_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); + uint32_t scaled_cores = uint32_t(std::ceil(cpu_info.cores() * cfg.feeding.concurrency)); // We need at least 1 guaranteed free worker in order to ensure progress. - return std::max(scaled_cores, (uint32_t)cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); + return std::max(scaled_cores, uint32_t(cfg.flush.maxconcurrent + 1u)); } uint32_t @@ -42,8 +42,8 @@ derive_warmup_threads(const HwInfo::Cpu& cpu_info) { uint32_t derive_field_writer_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) { - uint32_t scaled_cores = (size_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); - uint32_t field_writer_threads = std::max(scaled_cores, (uint32_t)cfg.indexing.threads); + uint32_t scaled_cores = size_t(std::ceil(cpu_info.cores() * cfg.feeding.concurrency)); + uint32_t field_writer_threads = std::max(scaled_cores, uint32_t(cfg.indexing.threads)); // Originally we used at least 3 threads for writing fields: // - index field inverter // - index field writer diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 9e65dfcfc07..e55344aded0 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -240,6 +240,24 @@ TEST_F("original lid range is used by read guard", Fixture) EXPECT_EQUAL(getUndefined<int>(), first_guard->getInt(DocId(10))); } +TEST_F("Original target lid range is used by read guard", Fixture) +{ + reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>( + f, BasicType::INT32, + {}); + EXPECT_EQUAL(11u, f.target_attr->getNumDocs()); + auto first_guard = f.get_imported_attr(); + add_n_docs_with_undefined_values(*f.target_attr, 1); + EXPECT_EQUAL(12u, f.target_attr->getNumDocs()); + auto typed_target_attr = f.template target_attr_as<IntegerAttribute>(); + ASSERT_TRUE(typed_target_attr->update(11, 2345)); + f.target_attr->commit(); + f.map_reference(DocId(8), dummy_gid(11), DocId(11)); + auto second_guard = f.get_imported_attr(); + EXPECT_EQUAL(2345, second_guard->getInt(DocId(8))); + EXPECT_NOT_EQUAL(2345, first_guard->getInt(DocId(8))); +} + struct SingleStringAttrFixture : Fixture { SingleStringAttrFixture() : Fixture() { setup(); diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 847a992d241..19327245083 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -429,6 +429,21 @@ TEST_F("original lid range is used by search context", SingleValueFixture) EXPECT_TRUE(second_ctx->matches(DocId(10))); } +TEST_F("Original target lid range is used by search context", SingleValueFixture) +{ + EXPECT_EQUAL(11u, f.target_attr->getNumDocs()); + auto first_ctx = f.create_context(word_term("2345")); + add_n_docs_with_undefined_values(*f.target_attr, 1); + EXPECT_EQUAL(12u, f.target_attr->getNumDocs()); + auto typed_target_attr = f.template target_attr_as<IntegerAttribute>(); + ASSERT_TRUE(typed_target_attr->update(11, 2345)); + f.target_attr->commit(); + f.map_reference(DocId(8), dummy_gid(11), DocId(11)); + auto second_ctx = f.create_context(word_term("2345")); + EXPECT_FALSE(first_ctx->matches(DocId(8))); + EXPECT_TRUE(second_ctx->matches(DocId(8))); +} + // Note: this uses an underlying string attribute, as queryTerm() does not seem to // implemented at all for (single) numeric attributes. Intentional? TEST_F("queryTerm() returns term context was created with", WsetValueFixture) { diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 2804e3f74e4..5ba90d2b077 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -390,7 +390,7 @@ TEST("testSingleValue") { EXPECT_EQUAL(24u, sizeof(SearchContext)); EXPECT_EQUAL(32u, sizeof(StringSearchHelper)); - EXPECT_EQUAL(80u, sizeof(attribute::SingleStringEnumSearchContext)); + EXPECT_EQUAL(88u, sizeof(attribute::SingleStringEnumSearchContext)); { Config cfg(BasicType::STRING, CollectionType::SINGLE); SingleValueStringAttribute svsa("svsa", cfg); diff --git a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp index 834cdbef50d..73a81be9f90 100644 --- a/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp +++ b/searchlib/src/tests/common/summaryfeatures/summaryfeatures.cpp @@ -2,9 +2,9 @@ #include <vespa/log/log.h> LOG_SETUP("summaryfeatures_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/featureset.h> +#include <vespa/vespalib/util/featureset.h> -using namespace search; +using vespalib::FeatureSet; using vespalib::Memory; TEST_SETUP(Test); diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index ae283f3f2b2..e5bed3ebae5 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -69,6 +69,8 @@ double computeEuclideanChecked(TypedCells a, TypedCells b) { return result; } +namespace { constexpr double sq_root_half = std::sqrt(0.5); } + TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { auto ct = vespalib::eval::CellType::DOUBLE; @@ -79,7 +81,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; @@ -179,7 +181,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; @@ -207,7 +209,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) EXPECT_DOUBLE_EQ(threshold, 0.5); double a34 = computeAngularChecked(t(p3), t(p4)); - EXPECT_FLOAT_EQ(a34, (1.0 - 0.707107)); + EXPECT_FLOAT_EQ(a34, (1.0 - sq_root_half)); EXPECT_FLOAT_EQ(angular->to_rawscore(a34), 1.0/(1.0 + pi/4)); threshold = angular->convert_threshold(pi/4); EXPECT_FLOAT_EQ(threshold, a34); @@ -257,6 +259,89 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) EXPECT_DOUBLE_EQ(a66, computeAngularChecked(t(iv6), t(iv6))); } +double computePrenormalizedAngularChecked(TypedCells a, TypedCells b) { + static PrenormalizedAngularDistanceFunctionFactory<float> flt_dff; + static PrenormalizedAngularDistanceFunctionFactory<double> dbl_dff; + auto d_n = dbl_dff.for_query_vector(a); + auto d_f = flt_dff.for_query_vector(a); + auto d_r = dbl_dff.for_query_vector(b); + auto d_i = dbl_dff.for_insertion_vector(a); + // normal: + double result = d_n->calc(b); + // insert is exactly same: + EXPECT_EQ(d_i->calc(b), result); + // note: for this distance, reverse is not necessarily equal, + // since we normalize based on length of LHS only + EXPECT_FLOAT_EQ(d_r->calc(a), result); + // float factory: + EXPECT_FLOAT_EQ(d_f->calc(b), result); + double closeness_n = d_n->to_rawscore(result); + double closeness_f = d_f->to_rawscore(result); + double closeness_r = d_r->to_rawscore(result); + double closeness_i = d_i->to_rawscore(result); + EXPECT_DOUBLE_EQ(closeness_n, closeness_f); + EXPECT_DOUBLE_EQ(closeness_n, closeness_r); + EXPECT_DOUBLE_EQ(closeness_n, closeness_i); + EXPECT_GT(closeness_n, 0.0); + EXPECT_LE(closeness_n, 1.0); + return result; +} + +TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) +{ + std::vector<double> p0{0.0, 0.0, 0.0}; + std::vector<double> p1{1.0, 0.0, 0.0}; + std::vector<double> p2{0.0, 1.0, 0.0}; + std::vector<double> p3{0.0, 0.0, 1.0}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; + std::vector<double> p5{0.0,-1.0, 0.0}; + std::vector<double> p6{1.0, 2.0, 2.0}; + std::vector<double> p7{2.0, -1.0, -2.0}; + std::vector<double> p8{3.0, 0.0, 0.0}; + + PrenormalizedAngularDistanceFunctionFactory<double> dff; + auto pnad = dff.for_query_vector(t(p0)); + + double i12 = computePrenormalizedAngularChecked(t(p1), t(p2)); + double i13 = computePrenormalizedAngularChecked(t(p1), t(p3)); + double i23 = computePrenormalizedAngularChecked(t(p2), t(p3)); + EXPECT_DOUBLE_EQ(i12, 1.0); + EXPECT_DOUBLE_EQ(i13, 1.0); + EXPECT_DOUBLE_EQ(i23, 1.0); + + double i14 = computePrenormalizedAngularChecked(t(p1), t(p4)); + double i24 = computePrenormalizedAngularChecked(t(p2), t(p4)); + EXPECT_DOUBLE_EQ(i14, 0.5); + EXPECT_DOUBLE_EQ(i24, 0.5); + double i34 = computePrenormalizedAngularChecked(t(p3), t(p4)); + EXPECT_FLOAT_EQ(i34, 1.0 - sq_root_half); + + double i25 = computePrenormalizedAngularChecked(t(p2), t(p5)); + EXPECT_DOUBLE_EQ(i25, 2.0); + + double i44 = computePrenormalizedAngularChecked(t(p4), t(p4)); + EXPECT_GE(i44, 0.0); + EXPECT_LT(i44, 0.000001); + + double i66 = computePrenormalizedAngularChecked(t(p6), t(p6)); + EXPECT_GE(i66, 0.0); + EXPECT_LT(i66, 0.000001); + + double i67 = computePrenormalizedAngularChecked(t(p6), t(p7)); + EXPECT_DOUBLE_EQ(i67, 13.0); + double i68 = computePrenormalizedAngularChecked(t(p6), t(p8)); + EXPECT_DOUBLE_EQ(i68, 6.0); + double i78 = computePrenormalizedAngularChecked(t(p7), t(p8)); + EXPECT_DOUBLE_EQ(i78, 3.0); + + double threshold = pnad->convert_threshold(0.25); + EXPECT_DOUBLE_EQ(threshold, 0.25); + threshold = pnad->convert_threshold(0.5); + EXPECT_DOUBLE_EQ(threshold, 0.5); + threshold = pnad->convert_threshold(1.0); + EXPECT_DOUBLE_EQ(threshold, 1.0); +} + TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) { auto ct = vespalib::eval::CellType::DOUBLE; @@ -267,7 +352,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; std::vector<double> p6{1.0, 2.0, 2.0}; @@ -283,7 +368,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) EXPECT_DOUBLE_EQ(i14, 0.5); EXPECT_DOUBLE_EQ(i24, 0.5); double i34 = innerproduct->calc(t(p3), t(p4)); - EXPECT_FLOAT_EQ(i34, 1.0 - 0.707107); + EXPECT_FLOAT_EQ(i34, 1.0 - sq_root_half); double i25 = innerproduct->calc(t(p2), t(p5)); EXPECT_DOUBLE_EQ(i25, 2.0); @@ -292,6 +377,10 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) EXPECT_GE(i44, 0.0); EXPECT_LT(i44, 0.000001); + double i66 = innerproduct->calc(t(p6), t(p6)); + EXPECT_GE(i66, 0.0); + EXPECT_LT(i66, 0.000001); + double threshold = innerproduct->convert_threshold(0.25); EXPECT_DOUBLE_EQ(threshold, 0.25); threshold = innerproduct->convert_threshold(0.5); diff --git a/searchlib/src/vespa/searchcommon/attribute/i_search_context.h b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h index 8867d1b87e4..4657d41a4a0 100644 --- a/searchlib/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h @@ -70,6 +70,11 @@ public: bool matches(DocId docId, int32_t &weight) const { return matches(*this, docId, weight); } bool matches(DocId doc) const { return find(doc, 0) >= 0; } + /* + * Committed docid limit on attribute vector when search context was + * created. + */ + virtual uint32_t get_committed_docid_limit() const noexcept = 0; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp index 7a5d82cd9ba..91bdb45ff19 100644 --- a/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/empty_search_context.cpp @@ -30,6 +30,12 @@ EmptySearchContext::approximateHits() const return 0u; } +uint32_t +EmptySearchContext::get_committed_docid_limit() const noexcept +{ + return 0u; +} + std::unique_ptr<queryeval::SearchIterator> EmptySearchContext::createIterator(fef::TermFieldMatchData*, bool) { diff --git a/searchlib/src/vespa/searchlib/attribute/empty_search_context.h b/searchlib/src/vespa/searchlib/attribute/empty_search_context.h index ae6f6d76edf..133e540d87f 100644 --- a/searchlib/src/vespa/searchlib/attribute/empty_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/empty_search_context.h @@ -19,6 +19,7 @@ class EmptySearchContext : public SearchContext public: EmptySearchContext(const AttributeVector& attr) noexcept; ~EmptySearchContext(); + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h index 0342976ffd6..86ffa1c8ab0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h @@ -41,6 +41,7 @@ protected: void fetchPostings(const queryeval::ExecuteInfo & execInfo) override; unsigned int approximateHits() const override; + uint32_t get_committed_docid_limit() const noexcept { return _docIdLimit; } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index a1a5e9f7894..b50a3720ff8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -17,12 +17,14 @@ ImportedAttributeVectorReadGuard::ImportedAttributeVectorReadGuard(std::shared_p _target_document_meta_store_read_guard(std::move(targetMetaStoreReadGuard)), _imported_attribute(imported_attribute), _targetLids(), + _target_docid_limit(0u), _reference_attribute_guard(imported_attribute.getReferenceAttribute()), _target_attribute_guard(imported_attribute.getTargetAttribute()->makeReadGuard(stableEnumGuard)), _reference_attribute(*imported_attribute.getReferenceAttribute()), _target_attribute(*_target_attribute_guard->attribute()) { _targetLids = _reference_attribute.getTargetLids(); + _target_docid_limit = _target_attribute.getCommittedDocIdLimit(); } ImportedAttributeVectorReadGuard::~ImportedAttributeVectorReadGuard() = default; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index cb48399f688..1297acad9b8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -95,6 +95,7 @@ private: std::shared_ptr<MetaStoreReadGuard> _target_document_meta_store_read_guard; const ImportedAttributeVector &_imported_attribute; TargetLids _targetLids; + uint32_t _target_docid_limit; AttributeGuard _reference_attribute_guard; std::unique_ptr<attribute::AttributeReadGuard> _target_attribute_guard; const ReferenceAttribute &_reference_attribute; @@ -103,7 +104,9 @@ protected: uint32_t getTargetLid(uint32_t lid) const { // Check range to avoid reading memory beyond end of mapping array - return lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + uint32_t target_lid = lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + // Check target range + return target_lid < _target_docid_limit ? target_lid : 0u; } long onSerializeForAscendingSort(DocId doc, void * serTo, long available, diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 1e8adc3922e..3d308b82b04 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -43,6 +43,7 @@ ImportedSearchContext::ImportedSearchContext( _target_attribute(target_attribute), _target_search_context(_target_attribute.createSearchContext(std::move(term), params)), _targetLids(_reference_attribute.getTargetLids()), + _target_docid_limit(_target_search_context->get_committed_docid_limit()), _merger(_reference_attribute.getCommittedDocIdLimit()), _params(params), _zero_hits(false) @@ -327,4 +328,10 @@ const vespalib::string& ImportedSearchContext::attributeName() const { return _imported_attribute.getName(); } +uint32_t +ImportedSearchContext::get_committed_docid_limit() const noexcept +{ + return _targetLids.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index d9c09d8c645..d6b6d09e8fc 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -39,6 +39,7 @@ class ImportedSearchContext : public ISearchContext { const IAttributeVector &_target_attribute; std::unique_ptr<ISearchContext> _target_search_context; TargetLids _targetLids; + uint32_t _target_docid_limit; PostingListMerger<int32_t> _merger; SearchContextParams _params; mutable std::atomic<bool> _zero_hits; @@ -47,7 +48,9 @@ class ImportedSearchContext : public ISearchContext { uint32_t getTargetLid(uint32_t lid) const { // Check range to avoid reading memory beyond end of mapping array - return lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + uint32_t target_lid = lid < _targetLids.size() ? _targetLids[lid].load_acquire() : 0u; + // Check target range + return target_lid < _target_docid_limit ? target_lid : 0u; } void makeMergedPostings(bool isFilter); @@ -90,6 +93,7 @@ public: const ISearchContext &target_search_context() const noexcept { return *_target_search_context; } + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h index 5b393d8bdb2..161c6799787 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.h @@ -59,6 +59,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp index e7901199e50..15abcf6f0d9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_enum_search_context.hpp @@ -33,4 +33,11 @@ MultiEnumSearchContext<T, BaseSC, M>::createFilterIterator(fef::TermFieldMatchDa : std::make_unique<AttributeIteratorT<MultiEnumSearchContext>>(*this, matchData); } +template <typename T, typename BaseSC, typename M> +uint32_t +MultiEnumSearchContext<T, BaseSC, M>::get_committed_docid_limit() const noexcept +{ + return _mv_mapping_read_view.get_committed_docid_limit(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h index b2c76a120f9..23e56e23af9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.h @@ -54,6 +54,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp index 15b851215f8..7e1fd1aeb5a 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_search_context.hpp @@ -33,4 +33,11 @@ MultiNumericSearchContext<T, M>::createFilterIterator(fef::TermFieldMatchData* m : std::make_unique<AttributeIteratorT<MultiNumericSearchContext<T, M>>>(*this, matchData); } +template <typename T, typename M> +uint32_t +MultiNumericSearchContext<T, M>::get_committed_docid_limit() const noexcept +{ + return _mv_mapping_read_view.get_committed_docid_limit(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h index 41138ff0890..609989208c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h @@ -33,6 +33,7 @@ public: } vespalib::ConstArrayRef<ElemT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } bool valid() const noexcept { return _store != nullptr; } + uint32_t get_committed_docid_limit() const noexcept { return _indices.size(); } }; } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index b701a6fd08f..9343dafe917 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -454,12 +454,14 @@ class ReferenceSearchContext : public attribute::SearchContext { private: const ReferenceAttribute& _ref_attr; GlobalId _term; + uint32_t _docid_limit; public: ReferenceSearchContext(const ReferenceAttribute& ref_attr, const GlobalId& term) : attribute::SearchContext(ref_attr), _ref_attr(ref_attr), - _term(term) + _term(term), + _docid_limit(ref_attr.getCommittedDocIdLimit()) { } bool valid() const override { @@ -480,8 +482,15 @@ public: int32_t weight; return onFind(docId, elementId, weight); } + uint32_t get_committed_docid_limit() const noexcept override; }; +uint32_t +ReferenceSearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } std::unique_ptr<attribute::SearchContext> diff --git a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h index 83d6c696117..f6a2f94dedb 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.h @@ -17,7 +17,9 @@ class SingleEnumSearchContext : public BaseSC { protected: using DocId = ISearchContext::DocId; - const vespalib::datastore::AtomicEntryRef* _enum_indices; + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using EnumIndices = vespalib::ConstArrayRef<AtomicEntryRef>; + EnumIndices _enum_indices; const EnumStoreT<T>& _enum_store; int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const final { @@ -29,7 +31,7 @@ protected: } public: - SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store); + SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store); int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { if ( elemId != 0) return -1; @@ -46,6 +48,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp index a415c301f9c..6b6cf480d6a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_enum_search_context.hpp @@ -9,7 +9,7 @@ namespace search::attribute { template <typename T, typename BaseSC> -SingleEnumSearchContext<T, BaseSC>::SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store) +SingleEnumSearchContext<T, BaseSC>::SingleEnumSearchContext(typename BaseSC::MatcherType&& matcher, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store) : BaseSC(toBeSearched, std::move(matcher)), _enum_indices(enum_indices), _enum_store(enum_store) @@ -33,4 +33,11 @@ SingleEnumSearchContext<T, BaseSC>::createFilterIterator(fef::TermFieldMatchData : std::make_unique<AttributeIteratorT<SingleEnumSearchContext>>(*this, matchData); } +template <typename T, typename BaseSC> +uint32_t +SingleEnumSearchContext<T, BaseSC>::get_committed_docid_limit() const noexcept +{ + return _enum_indices.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h index 86283f59283..fd3f4c03a8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.h @@ -16,7 +16,9 @@ template <typename T> class SingleNumericEnumSearchContext : public SingleEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>> { public: - SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store); + using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; + using EnumIndices = vespalib::ConstArrayRef<AtomicEntryRef>; + SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp index f4e049cb6f1..c0818d4d18a 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_enum_search_context.hpp @@ -8,7 +8,7 @@ namespace search::attribute { template <typename T> -SingleNumericEnumSearchContext<T>::SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<T>& enum_store) +SingleNumericEnumSearchContext<T>::SingleNumericEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<T>& enum_store) : SingleEnumSearchContext<T, NumericSearchContext<NumericRangeMatcher<T>>>(NumericRangeMatcher<T>(*qTerm, true), toBeSearched, enum_indices, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h index 5f6925f7f4d..6362c69cdac 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.h @@ -3,6 +3,7 @@ #pragma once #include "numeric_search_context.h" +#include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/util/atomic.h> namespace search::attribute { @@ -16,7 +17,7 @@ class SingleNumericSearchContext final : public NumericSearchContext<M> { private: using DocId = ISearchContext::DocId; - const T* _data; + vespalib::ConstArrayRef<T> _data; int32_t onFind(DocId docId, int32_t elemId, int32_t& weight) const override { return find(docId, elemId, weight); @@ -27,7 +28,7 @@ private: } public: - SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const T* data); + SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, vespalib::ConstArrayRef<T> data); int32_t find(DocId docId, int32_t elemId, int32_t& weight) const { if ( elemId != 0) return -1; const T v = vespalib::atomic::load_ref_relaxed(_data[docId]); @@ -43,6 +44,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp index 75d3da9de7f..b40b1336e6f 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp +++ b/searchlib/src/vespa/searchlib/attribute/single_numeric_search_context.hpp @@ -9,7 +9,7 @@ namespace search::attribute { template <typename T, typename M> -SingleNumericSearchContext<T, M>::SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const T* data) +SingleNumericSearchContext<T, M>::SingleNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, vespalib::ConstArrayRef<T> data) : NumericSearchContext<M>(toBeSearched, *qTerm, true), _data(data) { @@ -32,4 +32,11 @@ SingleNumericSearchContext<T, M>::createFilterIterator(fef::TermFieldMatchData* : std::make_unique<AttributeIteratorT<SingleNumericSearchContext<T, M>>>(*this, matchData); } +template <typename T, typename M> +uint32_t +SingleNumericSearchContext<T, M>::get_committed_docid_limit() const noexcept +{ + return _data.size(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp index 5eeef7cd61a..074435809cc 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.cpp @@ -6,13 +6,14 @@ namespace search::attribute { -SingleSmallNumericSearchContext::SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift) +SingleSmallNumericSearchContext::SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift, uint32_t docid_limit) : NumericSearchContext<NumericRangeMatcher<T>>(toBeSearched, *qTerm, false), _wordData(word_data), _valueMask(value_mask), _valueShiftShift(value_shift_shift), _valueShiftMask(value_shift_mask), - _wordShift(word_shift) + _wordShift(word_shift), + _docid_limit(docid_limit) { } @@ -32,4 +33,10 @@ SingleSmallNumericSearchContext::createFilterIterator(fef::TermFieldMatchData* m : std::make_unique<AttributeIteratorT<SingleSmallNumericSearchContext>>(*this, matchData); } +uint32_t +SingleSmallNumericSearchContext::get_committed_docid_limit() const noexcept +{ + return _docid_limit; +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h index 46ed02b3eca..a42c8b9b29c 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_small_numeric_search_context.h @@ -22,6 +22,7 @@ private: uint32_t _valueShiftShift; uint32_t _valueShiftMask; uint32_t _wordShift; + uint32_t _docid_limit; int32_t onFind(DocId docId, int32_t elementId, int32_t & weight) const override { return find(docId, elementId, weight); @@ -32,7 +33,7 @@ private: } public: - SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift); + SingleSmallNumericSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, const Word* word_data, Word value_mask, uint32_t value_shift_shift, uint32_t value_shift_mask, uint32_t word_shift, uint32_t docid_limit); int32_t find(DocId docId, int32_t elemId, int32_t & weight) const { if ( elemId != 0) return -1; @@ -53,6 +54,7 @@ public: std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData* matchData, bool strict) override; + uint32_t get_committed_docid_limit() const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp index 70023b27802..2d1748cefa5 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.cpp @@ -5,10 +5,10 @@ namespace search::attribute { -SingleStringEnumHintSearchContext::SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values) +SingleStringEnumHintSearchContext::SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store, uint64_t num_values) : SingleStringEnumSearchContext(std::move(qTerm), cased, toBeSearched, enum_indices, enum_store), EnumHintSearchContext(enum_store.get_dictionary(), - doc_id_limit, num_values) + enum_indices.size(), num_values) { setup_enum_hint_sc(enum_store, *this); } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h index f9d44454cd0..f157bf17a71 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_hint_search_context.h @@ -16,7 +16,7 @@ class SingleStringEnumHintSearchContext : public SingleStringEnumSearchContext, public EnumHintSearchContext { public: - SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store, uint32_t doc_id_limit, uint64_t num_values); + SingleStringEnumHintSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store, uint64_t num_values); ~SingleStringEnumHintSearchContext() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp index cba1d207501..8d23eaf7af0 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.cpp @@ -6,7 +6,7 @@ namespace search::attribute { -SingleStringEnumSearchContext::SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store) +SingleStringEnumSearchContext::SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store) : SingleEnumSearchContext<const char*, StringSearchContext>(StringMatcher(std::move(qTerm), cased), toBeSearched, enum_indices, enum_store) { } diff --git a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h index 6a9ed38b4ea..b8014b1b0e3 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/single_string_enum_search_context.h @@ -14,7 +14,7 @@ namespace search::attribute { class SingleStringEnumSearchContext : public SingleEnumSearchContext<const char*, StringSearchContext> { public: - SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, const vespalib::datastore::AtomicEntryRef* enum_indices, const EnumStoreT<const char*>& enum_store); + SingleStringEnumSearchContext(std::unique_ptr<QueryTermSimple> qTerm, bool cased, const AttributeVector& toBeSearched, EnumIndices enum_indices, const EnumStoreT<const char*>& enum_store); SingleStringEnumSearchContext(SingleStringEnumSearchContext&&) noexcept; ~SingleStringEnumSearchContext() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 15fc819300c..87b7049b9b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -132,6 +132,7 @@ public: void fetchPostings(const queryeval::ExecuteInfo &execInfo) override; std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; unsigned int approximateHits() const override; + uint32_t get_committed_docid_limit() const noexcept override; }; BitVectorSearchContext::BitVectorSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const SingleBoolAttribute & attr) @@ -177,6 +178,12 @@ BitVectorSearchContext::approximateHits() const { : 0; } +uint32_t +BitVectorSearchContext::get_committed_docid_limit() const noexcept +{ + return _doc_id_limit; +} + } std::unique_ptr<attribute::SearchContext> diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index c75ee0aacb5..606c7a92ef5 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -164,7 +164,7 @@ SingleValueNumericAttribute<B>::getSearch(QueryTermSimple::UP qTerm, { (void) params; QueryTermSimple::RangeResult<T> res = qTerm->getRange<T>(); - const T* data = &_data.acquire_elem_ref(0); + auto data = _data.make_read_view(this->getCommittedDocIdLimit()); if (res.isEqual()) { return std::make_unique<attribute::SingleNumericSearchContext<T, attribute::NumericMatcher<T>>>(std::move(qTerm), *this, data); } else { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index b840a0516b2..e459d3d9c9c 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -160,7 +160,8 @@ SingleValueNumericEnumAttribute<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams & params) const { (void) params; - return std::make_unique<attribute::SingleNumericEnumSearchContext<T>>(std::move(qTerm), *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleNumericEnumSearchContext<T>>(std::move(qTerm), *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index e353d03a9e8..a4b9abb084a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -143,7 +143,8 @@ SingleValueNumericPostingAttribute<B>::getSearch(QueryTermSimple::UP qTerm, { using BaseSC = attribute::SingleNumericEnumSearchContext<T>; using SC = attribute::NumericPostingSearchContext<BaseSC, SelfType, vespalib::btree::BTreeNoLeafData>; - BaseSC base_sc(std::move(qTerm), *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params, *this); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 13bf2f932e8..3c1621ac244 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -170,7 +170,8 @@ std::unique_ptr<attribute::SearchContext> SingleValueSmallNumericAttribute::getSearch(std::unique_ptr<QueryTermSimple> qTerm, const attribute::SearchContextParams &) const { - return std::make_unique<attribute::SingleSmallNumericSearchContext>(std::move(qTerm), *this, &_wordData.acquire_elem_ref(0), _valueMask, _valueShiftShift, _valueShiftMask, _wordShift); + auto docid_limit = getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleSmallNumericSearchContext>(std::move(qTerm), *this, &_wordData.acquire_elem_ref(0), _valueMask, _valueShiftShift, _valueShiftMask, _wordShift, docid_limit); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 69fe6435a03..c3f5c295260 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -46,7 +46,8 @@ SingleValueStringAttributeT<B>::getSearch(QueryTermSimpleUP qTerm, const attribute::SearchContextParams &) const { bool cased = this->get_match_is_cased(); - return std::make_unique<attribute::SingleStringEnumHintSearchContext>(std::move(qTerm), cased, *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore, this->getCommittedDocIdLimit(), this->getStatus().getNumValues()); + auto docid_limit = this->getCommittedDocIdLimit(); + return std::make_unique<attribute::SingleStringEnumHintSearchContext>(std::move(qTerm), cased, *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore, this->getStatus().getNumValues()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index 5b5214f6d3e..60847636baa 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -145,7 +145,8 @@ SingleValueStringPostingAttributeT<B>::getSearch(QueryTermSimpleUP qTerm, using BaseSC = attribute::SingleStringEnumSearchContext; using SC = attribute::StringPostingSearchContext<BaseSC, SelfType, vespalib::btree::BTreeNoLeafData>; bool cased = this->get_match_is_cased(); - BaseSC base_sc(std::move(qTerm), cased, *this, &this->_enumIndices.acquire_elem_ref(0), this->_enumStore); + auto docid_limit = this->getCommittedDocIdLimit(); + BaseSC base_sc(std::move(qTerm), cased, *this, this->_enumIndices.make_read_view(docid_limit), this->_enumStore); return std::make_unique<SC>(std::move(base_sc), params.useBitVector(), *this); diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index a7c8d56f11d..089151455f3 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_add_library(searchlib_common OBJECT condensedbitvectors.cpp documentlocations.cpp documentsummary.cpp - featureset.cpp fileheadercontext.cpp flush_token.cpp geo_gcd.cpp diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index 8f862d8dcf7..6b0edca3086 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -6,8 +6,8 @@ #include <vespa/document/base/globalid.h> #include <vespa/searchlib/common/hitrank.h> #include <vespa/searchlib/common/unique_issues.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/featureset.h> #include <vector> namespace search::engine { @@ -15,6 +15,7 @@ namespace search::engine { class SearchReply { public: + using FeatureValues = vespalib::FeatureValues; using UP = std::unique_ptr<SearchReply>; class Hit diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index c50c6ec49f5..86f520c8711 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -129,8 +129,16 @@ struct FakeContext : attribute::ISearchContext { DoubleRange getAsDoubleTerm() const override { abort(); } const QueryTermUCS4 * queryTerm() const override { abort(); } const vespalib::string &attributeName() const override { return name; } + uint32_t get_committed_docid_limit() const noexcept override; }; +uint32_t +FakeContext::get_committed_docid_limit() const noexcept +{ + auto& documents = result.inspect(); + return documents.empty() ? 0 : (documents.back().docId + 1); +} + } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 1783e0da1dd..2e874ffa4ae 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -30,6 +30,7 @@ vespa_add_library(searchlib_tensor OBJECT large_subspaces_buffer_type.cpp nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp + prenormalized_angular_distance.cpp serialized_fast_value_attribute.cpp serialized_tensor_ref.cpp small_subspaces_buffer_type.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index 85eac76728c..efc1170baf5 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -66,15 +66,15 @@ public: _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) { - auto a = &_lhs[0]; + auto a = _lhs.data(); _lhs_norm_sq = _computer.dotProduct(a, a, lhs.size); } double calc(const vespalib::eval::TypedCells& rhs) const override { size_t sz = _lhs.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); assert(sz == rhs_vector.size()); - auto a = &_lhs[0]; - auto b = &rhs_vector[0]; + auto a = _lhs.data(); + auto b = rhs_vector.data(); double b_norm_sq = _computer.dotProduct(b, b, sz); double squared_norms = _lhs_norm_sq * b_norm_sq; double dot_product = _computer.dotProduct(a, b, sz); diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.h b/searchlib/src/vespa/searchlib/tensor/angular_distance.h index 97f692d05a2..bba83576153 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.h @@ -60,8 +60,8 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); - auto a = &lhs_vector[0]; - auto b = &rhs_vector[0]; + auto a = lhs_vector.data(); + auto b = rhs_vector.data(); double a_norm_sq = _computer.dotProduct(a, a, sz); double b_norm_sq = _computer.dotProduct(b, b, sz); double squared_norms = a_norm_sq * b_norm_sq; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index cca492ef212..4553f39a525 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -107,6 +107,20 @@ make_distance_function_factory(search::attribute::DistanceMetric variant, default: return std::make_unique<EuclideanDistanceFunctionFactory<float>>(); } } + if (variant == DistanceMetric::PrenormalizedAngular) { + if (cell_type == CellType::DOUBLE) { + return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<double>>(); + } + return std::make_unique<PrenormalizedAngularDistanceFunctionFactory<float>>(); + } + /* + if (variant == DistanceMetric::GeoDegrees) { + return std::make_unique<GeoDistanceFunctionFactory>(); + } + if (variant == DistanceMetric::Hamming) { + return std::make_unique<HammingDistanceFunctionFactory>(); + } + */ auto df = make_distance_function(variant, cell_type); return std::make_unique<SimpleDistanceFunctionFactory>(std::move(df)); } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index b28cc2bda46..2300dba2db1 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -8,3 +8,4 @@ #include "geo_degrees_distance.h" #include "hamming_distance.h" #include "inner_product_distance.h" +#include "prenormalized_angular_distance.h" diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index 92d4e7af406..9c37b191637 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -71,8 +71,8 @@ public: size_t sz = _lhs_vector.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); assert(sz == rhs_vector.size()); - auto a = &_lhs_vector[0]; - auto b = &rhs_vector[0]; + auto a = _lhs_vector.data(); + auto b = rhs_vector.data(); return _computer.squaredEuclideanDistance(cast(a), cast(b), sz); } double convert_threshold(double threshold) const override { diff --git a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h index 8ba14580885..135bb186fd4 100644 --- a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h @@ -54,7 +54,7 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); - double score = 1.0 - _computer.dotProduct(&lhs_vector[0], &rhs_vector[0], sz); + double score = 1.0 - _computer.dotProduct(lhs_vector.data(), rhs_vector.data(), sz); return std::max(0.0, score); } private: diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp new file mode 100644 index 00000000000..d2693f9f443 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "prenormalized_angular_distance.h" +#include "temporary_vector_store.h" + +using vespalib::typify_invoke; +using vespalib::eval::TypifyCellType; + +namespace search::tensor { + +template<typename FloatType> +class BoundPrenormalizedAngularDistance : public BoundDistanceFunction { +private: + const vespalib::hwaccelrated::IAccelrated & _computer; + mutable TemporaryVectorStore<FloatType> _tmpSpace; + const vespalib::ConstArrayRef<FloatType> _lhs; + double _lhs_norm_sq; +public: + BoundPrenormalizedAngularDistance(const vespalib::eval::TypedCells& lhs) + : BoundDistanceFunction(vespalib::eval::get_cell_type<FloatType>()), + _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), + _tmpSpace(lhs.size), + _lhs(_tmpSpace.storeLhs(lhs)) + { + auto a = _lhs.data(); + _lhs_norm_sq = _computer.dotProduct(a, a, lhs.size); + if (_lhs_norm_sq <= 0.0) { + _lhs_norm_sq = 1.0; + } + } + double calc(const vespalib::eval::TypedCells& rhs) const override { + size_t sz = _lhs.size(); + vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); + assert(sz == rhs_vector.size()); + auto a = _lhs.data(); + auto b = rhs_vector.data(); + double dot_product = _computer.dotProduct(a, b, sz); + double distance = _lhs_norm_sq - dot_product; + return distance; + } + double convert_threshold(double threshold) const override { + double cosine_similarity = 1.0 - threshold; + double dot_product = cosine_similarity * _lhs_norm_sq; + double distance = _lhs_norm_sq - dot_product; + return distance; + } + double to_rawscore(double distance) const override { + double dot_product = _lhs_norm_sq - distance; + double cosine_similarity = dot_product / _lhs_norm_sq; + // should be in in range [-1,1] but roundoff may cause problems: + cosine_similarity = std::min(1.0, cosine_similarity); + cosine_similarity = std::max(-1.0, cosine_similarity); + double cosine_distance = 1.0 - cosine_similarity; // in range [0,2] + double score = 1.0 / (1.0 + cosine_distance); + return score; + } + double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + return calc(rhs); + } +}; + +template class BoundPrenormalizedAngularDistance<float>; +template class BoundPrenormalizedAngularDistance<double>; + +template <typename FloatType> +BoundDistanceFunction::UP +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundPrenormalizedAngularDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template <typename FloatType> +BoundDistanceFunction::UP +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { + using DFT = BoundPrenormalizedAngularDistance<FloatType>; + return std::make_unique<DFT>(lhs); +} + +template class PrenormalizedAngularDistanceFunctionFactory<float>; +template class PrenormalizedAngularDistanceFunctionFactory<double>; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h new file mode 100644 index 00000000000..88953a236e7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "distance_function.h" +#include "bound_distance_function.h" +#include "distance_function_factory.h" +#include <vespa/eval/eval/typed_cells.h> +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> + +namespace search::tensor { + +/** + * Calculates inner-product "distance" between vectors with assumed norm 1. + * Should give same ordering as Angular distance, but is less expensive. + */ +template <typename FloatType> +class PrenormalizedAngularDistanceFunctionFactory : public DistanceFunctionFactory { +public: + PrenormalizedAngularDistanceFunctionFactory() + : DistanceFunctionFactory(vespalib::eval::get_cell_type<FloatType>()) + {} + BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 2c644a243c8..a765208cb9e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -3,9 +3,9 @@ #pragma once #include "getdocsumargs.h" -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/stash.h> namespace juniper { @@ -48,6 +48,7 @@ protected: class GetDocsumsState { public: + using FeatureSet = vespalib::FeatureSet; const search::attribute::IAttributeVector * getAttribute(size_t index) const { return _attributes[index]; } GetDocsumArgs _args; // from getdocsums request diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index bad1ad5a6f3..c5e823bf9f4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -5,6 +5,8 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> +using vespalib::FeatureSet; + namespace search::docsummary { RankFeaturesDFW::RankFeaturesDFW() = default; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index a680b01d887..a1b2d6b3af6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -5,8 +5,7 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.docsummary.summaryfeaturesdfw"); +using vespalib::FeatureSet; namespace search::docsummary { diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp index 6950c90f097..791ec01162f 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp @@ -285,7 +285,7 @@ HitCollectorTest::testFeatureSet() FeatureResolver resolver(rankProgram.get_resolver()); search::StringStringMap renames; renames["bar"] = "qux"; - search::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver, renames); + vespalib::FeatureSet::SP sf = hc.getFeatureSet(rankProgram, resolver, renames); EXPECT_EQUAL(sf->getNames().size(), 3u); EXPECT_EQUAL(sf->getNames()[0], "foo"); diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index 10e6c6aa68a..7b4e3cb0208 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -10,8 +10,8 @@ #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.hitcollector"); -using search::FeatureSet; using search::fef::MatchData; +using vespalib::FeatureSet; using vdslib::SearchResult; namespace streaming { diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index 6ce7459adfd..2918f815811 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -2,13 +2,13 @@ #pragma once -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchlib/common/stringmap.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/vdslib/container/searchresult.h> #include <vespa/vsm/common/docsum.h> #include <vespa/vsm/common/storagedocument.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/featureset.h> namespace search { namespace fef { class FeatureResolver; } } @@ -132,9 +132,9 @@ public: * @param rankProgram the rank program used to calculate all features. * @param resolver feature resolver, gives feature names and values **/ - search::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, - const search::fef::FeatureResolver &resolver, - const search::StringStringMap &feature_rename_map); + vespalib::FeatureSet::SP getFeatureSet(IRankProgram &rankProgram, + const search::fef::FeatureResolver &resolver, + const search::StringStringMap &feature_rename_map); }; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 3751ba379d0..3ce137bffe5 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -10,7 +10,7 @@ #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.rankprocessor"); -using search::FeatureSet; +using vespalib::FeatureSet; using search::fef::FeatureHandle; using search::fef::ITermData; using search::fef::ITermFieldData; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index c541f62646e..c74a2d1e3ee 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -65,7 +65,7 @@ public: void unpackMatchData(uint32_t docId); static void unpack_match_data(uint32_t docid, search::fef::MatchData& matchData, QueryWrapper& query); void runRankProgram(uint32_t docId); - search::FeatureSet::SP calculateFeatureSet(); + vespalib::FeatureSet::SP calculateFeatureSet(); void fillSearchResult(vdslib::SearchResult & searchResult); const search::fef::MatchData &getMatchData() const { return *_match_data; } void setRankScore(double score) { _score = score; } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 7dc0c05cfaa..8980bc1f54d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -664,14 +664,14 @@ SearchVisitor::RankController::onCompletedVisiting(vsm::GetDocsumsStateCallback // calculate summary features and set them on the callback object if (!_rankSetup->getSummaryFeatures().empty()) { LOG(debug, "Calculate summary features"); - search::FeatureSet::SP sf = _rankProcessor->calculateFeatureSet(); + vespalib::FeatureSet::SP sf = _rankProcessor->calculateFeatureSet(); docsumsStateCallback.setSummaryFeatures(sf); } // calculate rank features and set them on the callback object if (_dumpFeatures) { LOG(debug, "Calculate rank features"); - search::FeatureSet::SP rf = _dumpProcessor->calculateFeatureSet(); + vespalib::FeatureSet::SP rf = _dumpProcessor->calculateFeatureSet(); docsumsStateCallback.setRankFeatures(rf); } } diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index 77ed9573e54..ba87ccfef05 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -5,11 +5,11 @@ #include <vespa/searchlib/query/base.h> #include <vespa/vsm/config/vsm-cfif.h> #include <vespa/config-summary.h> -#include <vespa/searchlib/common/featureset.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/juniper/rpinterface.h> +#include <vespa/vespalib/util/featureset.h> using search::docsummary::ResultConfig; using search::docsummary::ResultClass; @@ -28,8 +28,8 @@ class IMatchingElementsFiller; class GetDocsumsStateCallback : public search::docsummary::GetDocsumsStateCallback { private: - search::FeatureSet::SP _summaryFeatures; - search::FeatureSet::SP _rankFeatures; + vespalib::FeatureSet::SP _summaryFeatures; + vespalib::FeatureSet::SP _rankFeatures; std::unique_ptr<IMatchingElementsFiller> _matching_elements_filler; public: @@ -37,8 +37,8 @@ public: void fillSummaryFeatures(GetDocsumsState& state) override; void fillRankFeatures(GetDocsumsState& state) override; std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; - void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } - void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } + void setSummaryFeatures(const vespalib::FeatureSet::SP & sf) { _summaryFeatures = sf; } + void setRankFeatures(const vespalib::FeatureSet::SP & rf) { _rankFeatures = rf; } void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler); ~GetDocsumsStateCallback() override; }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java deleted file mode 100644 index c2ab22f4921..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/DefaultSignedIdentityDocument.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -public record DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, - String data, IdentityDocument identityDocument) implements SignedIdentityDocument { - - public DefaultSignedIdentityDocument { - identityDocument = EntityBindingsMapper.fromIdentityDocumentData(data); - } - - public DefaultSignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, String data) { - this(signature,signingKeyVersion,documentVersion, data, null); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index a695e10a29c..2d77d2ceda1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -6,12 +6,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.DefaultSignedIdentityDocumentEntity; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentEntity; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.LegacySignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; @@ -20,8 +16,6 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.time.Instant; -import java.util.Base64; import java.util.Optional; import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.fromDottedString; @@ -30,7 +24,6 @@ import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId. * Utility class for mapping objects model types and their Jackson binding versions. * * @author bjorncs - * @author mortent */ public class EntityBindingsMapper { @@ -55,60 +48,39 @@ public class EntityBindingsMapper { } public static SignedIdentityDocument toSignedIdentityDocument(SignedIdentityDocumentEntity entity) { - if (entity instanceof LegacySignedIdentityDocumentEntity docEntity) { - IdentityDocument doc = new IdentityDocument( - fromDottedString(docEntity.providerUniqueId()), - new AthenzService(docEntity.providerService()), - docEntity.configServerHostname(), - docEntity.instanceHostname(), - docEntity.createdAt(), - docEntity.ipAddresses(), - IdentityType.fromId(docEntity.identityType()), - Optional.ofNullable(docEntity.clusterType()).map(ClusterType::from).orElse(null), - docEntity.ztsUrl(), - Optional.ofNullable(docEntity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), - docEntity.unknownAttributes()); - return new LegacySignedIdentityDocument( - docEntity.signature(), - docEntity.signingKeyVersion(), - entity.documentVersion(), - doc); - } else if (entity instanceof DefaultSignedIdentityDocumentEntity docEntity) { - return new DefaultSignedIdentityDocument(docEntity.signature(), - docEntity.signingKeyVersion(), - docEntity.documentVersion(), - docEntity.data()); - } else { - throw new IllegalArgumentException("Unknown signed identity document type: " + entity.getClass().getName()); - } + return new SignedIdentityDocument( + entity.signature(), + entity.signingKeyVersion(), + fromDottedString(entity.providerUniqueId()), + new AthenzService(entity.providerService()), + entity.documentVersion(), + entity.configServerHostname(), + entity.instanceHostname(), + entity.createdAt(), + entity.ipAddresses(), + IdentityType.fromId(entity.identityType()), + Optional.ofNullable(entity.clusterType()).map(ClusterType::from).orElse(null), + entity.ztsUrl(), + Optional.ofNullable(entity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), + entity.unknownAttributes()); } public static SignedIdentityDocumentEntity toSignedIdentityDocumentEntity(SignedIdentityDocument model) { - if (model instanceof LegacySignedIdentityDocument legacyModel) { - IdentityDocument idDoc = legacyModel.identityDocument(); - return new LegacySignedIdentityDocumentEntity( - legacyModel.signature(), - legacyModel.signingKeyVersion(), - idDoc.providerUniqueId().asDottedString(), - idDoc.providerService().getFullName(), - legacyModel.documentVersion(), - idDoc.configServerHostname(), - idDoc.instanceHostname(), - idDoc.createdAt(), - idDoc.ipAddresses(), - idDoc.identityType().id(), - Optional.ofNullable(idDoc.clusterType()).map(ClusterType::toConfigValue).orElse(null), - idDoc.ztsUrl(), - Optional.ofNullable(idDoc.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), - idDoc.unknownAttributes()); - } else if (model instanceof DefaultSignedIdentityDocument defaultModel){ - return new DefaultSignedIdentityDocumentEntity(defaultModel.signature(), - defaultModel.signingKeyVersion(), - defaultModel.documentVersion(), - defaultModel.data()); - } else { - throw new IllegalArgumentException("Unsupported model type: " + model.getClass().getName()); - } + return new SignedIdentityDocumentEntity( + model.signature(), + model.signingKeyVersion(), + model.providerUniqueId().asDottedString(), + model.providerService().getFullName(), + model.documentVersion(), + model.configServerHostname(), + model.instanceHostname(), + model.createdAt(), + model.ipAddresses(), + model.identityType().id(), + Optional.ofNullable(model.clusterType()).map(ClusterType::toConfigValue).orElse(null), + model.ztsUrl(), + Optional.ofNullable(model.serviceIdentity()).map(AthenzIdentity::getFullName).orElse(null), + model.unknownAttributes()); } public static SignedIdentityDocument readSignedIdentityDocumentFromFile(Path file) { @@ -132,40 +104,4 @@ public class EntityBindingsMapper { } } - public static IdentityDocument fromIdentityDocumentData(String data) { - byte[] decoded = Base64.getDecoder().decode(data); - IdentityDocumentEntity docEntity = Exceptions.uncheck(() -> mapper.readValue(decoded, IdentityDocumentEntity.class)); - return new IdentityDocument( - fromDottedString(docEntity.providerUniqueId()), - new AthenzService(docEntity.providerService()), - docEntity.configServerHostname(), - docEntity.instanceHostname(), - docEntity.createdAt(), - docEntity.ipAddresses(), - IdentityType.fromId(docEntity.identityType()), - Optional.ofNullable(docEntity.clusterType()).map(ClusterType::from).orElse(null), - docEntity.ztsUrl(), - Optional.ofNullable(docEntity.serviceIdentity()).map(AthenzIdentities::from).orElse(null), - docEntity.unknownAttributes()); - } - - public static String toIdentityDocmentData(IdentityDocument identityDocument) { - IdentityDocumentEntity documentEntity = new IdentityDocumentEntity( - identityDocument.providerUniqueId().asDottedString(), - identityDocument.providerService().getFullName(), - identityDocument.configServerHostname(), - identityDocument.instanceHostname(), - identityDocument.createdAt(), - identityDocument.ipAddresses(), - identityDocument.identityType().id(), - Optional.ofNullable(identityDocument.clusterType()).map(ClusterType::toConfigValue).orElse(null), - identityDocument.ztsUrl(), - identityDocument.serviceIdentity().getFullName()); - try { - byte[] bytes = mapper.writeValueAsBytes(documentEntity); - return Base64.getEncoder().encodeToString(bytes); - } catch (JsonProcessingException e) { - throw new RuntimeException("Error during serialization of identity document.", e); - } - } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java deleted file mode 100644 index 577584db185..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Represents an unsigned identity document - * @author mortent - */ -public record IdentityDocument(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, - String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, - AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { - - public IdentityDocument { - ipAddresses = Set.copyOf(ipAddresses); - - Map<String, Object> nonNull = new HashMap<>(); - unknownAttributes.forEach((key, value) -> { - if (value != null) nonNull.put(key, value); - }); - // Map.copyOf() does not allow null values - unknownAttributes = Map.copyOf(nonNull); - } - - public IdentityDocument(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, - String instanceHostname, Instant createdAt, Set<String> ipAddresses, - IdentityType identityType, ClusterType clusterType, String ztsUrl, - AthenzIdentity serviceIdentity) { - this(providerUniqueId, providerService, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); - } - - - public IdentityDocument withServiceIdentity(AthenzService athenzService) { - return new IdentityDocument( - this.providerUniqueId, - this.providerService, - this.configServerHostname, - this.instanceHostname, - this.createdAt, - this.ipAddresses, - this.identityType, - this.clusterType, - this.ztsUrl, - athenzService, - this.unknownAttributes); - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java index 0e13cba8de9..5a0f77ec765 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocumentClient.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; -import java.util.OptionalInt; - /** * A client that communicates that fetches an identity document. * * @author bjorncs */ public interface IdentityDocumentClient { - SignedIdentityDocument getNodeIdentityDocument(String host, int documentVersion); - SignedIdentityDocument getTenantIdentityDocument(String host, int documentVersion); + SignedIdentityDocument getNodeIdentityDocument(String host); + SignedIdentityDocument getTenantIdentityDocument(String host); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java deleted file mode 100644 index 220bc72a017..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/LegacySignedIdentityDocument.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api; - -public record LegacySignedIdentityDocument(String signature, int signingKeyVersion, int documentVersion, - IdentityDocument identityDocument) implements SignedIdentityDocument { -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 4e3bd8dee91..de78d81cd1b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -1,20 +1,54 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; + +import java.net.URL; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + /** * A signed identity document. + * The {@link #unknownAttributes()} member provides forward compatibility and ensures any new/unknown fields are kept intact when serialized to JSON. + * * @author bjorncs - * @author mortent */ -public interface SignedIdentityDocument { +public record SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, + AthenzService providerService, int documentVersion, String configServerHostname, + String instanceHostname, Instant createdAt, Set<String> ipAddresses, + IdentityType identityType, ClusterType clusterType, String ztsUrl, + AthenzIdentity serviceIdentity, Map<String, Object> unknownAttributes) { + + public SignedIdentityDocument { + ipAddresses = Set.copyOf(ipAddresses); + + Map<String, Object> nonNull = new HashMap<>(); + unknownAttributes.forEach((key, value) -> { + if (value != null) nonNull.put(key, value); + }); + // Map.copyOf() does not allow null values + unknownAttributes = Map.copyOf(nonNull); + } + + public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, + AthenzService providerService, int documentVersion, String configServerHostname, + String instanceHostname, Instant createdAt, Set<String> ipAddresses, + IdentityType identityType, ClusterType clusterType, String ztsUrl, AthenzIdentity serviceIdentity) { + this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, Map.of()); + } + + public static final int DEFAULT_DOCUMENT_VERSION = 3; + + public boolean outdated() { return documentVersion < DEFAULT_DOCUMENT_VERSION; } - int LEGACY_DEFAULT_DOCUMENT_VERSION = 3; - int DEFAULT_DOCUMENT_VERSION = 4; + public SignedIdentityDocument withServiceIdentity(AthenzIdentity identity) { + return new SignedIdentityDocument(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, instanceHostname, createdAt, + ipAddresses, identityType, clusterType, ztsUrl, identity); + } - default boolean outdated() { return documentVersion() < LEGACY_DEFAULT_DOCUMENT_VERSION; } - IdentityDocument identityDocument(); - String signature(); - int signingKeyVersion(); - int documentVersion(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java deleted file mode 100644 index 3aaff011415..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/DefaultSignedIdentityDocumentEntity.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public record DefaultSignedIdentityDocumentEntity( - @JsonProperty("signature") String signature, - @JsonProperty("signing-key-version") int signingKeyVersion, - @JsonProperty("document-version") int documentVersion, - @JsonProperty("data") String data) - implements SignedIdentityDocumentEntity { -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java deleted file mode 100644 index 946eacc67eb..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author bjorncs - * @author mortent - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record IdentityDocumentEntity(String providerUniqueId, String providerService, - String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { - - @JsonCreator - public IdentityDocumentEntity(@JsonProperty("provider-unique-id") String providerUniqueId, - @JsonProperty("provider-service") String providerService, - @JsonProperty("configserver-hostname") String configServerHostname, - @JsonProperty("instance-hostname") String instanceHostname, - @JsonProperty("created-at") Instant createdAt, - @JsonProperty("ip-addresses") Set<String> ipAddresses, - @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl, - @JsonProperty("service-identity") String serviceIdentity) { - this(providerUniqueId, providerService, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); - } - - @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } - @JsonProperty("provider-service") @Override public String providerService() { return providerService; } - @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } - @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } - @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } - @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } - @JsonProperty("identity-type") @Override public String identityType() { return identityType; } - @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } - @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } - @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } - @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java deleted file mode 100644 index e00ab9978f6..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/LegacySignedIdentityDocumentEntity.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author bjorncs - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record LegacySignedIdentityDocumentEntity ( - String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, - String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, - String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) implements SignedIdentityDocumentEntity { - - @JsonCreator - public LegacySignedIdentityDocumentEntity(@JsonProperty("signature") String signature, - @JsonProperty("signing-key-version") int signingKeyVersion, - @JsonProperty("provider-unique-id") String providerUniqueId, - @JsonProperty("provider-service") String providerService, - @JsonProperty("document-version") int documentVersion, - @JsonProperty("configserver-hostname") String configServerHostname, - @JsonProperty("instance-hostname") String instanceHostname, - @JsonProperty("created-at") Instant createdAt, - @JsonProperty("ip-addresses") Set<String> ipAddresses, - @JsonProperty("identity-type") String identityType, - @JsonProperty("cluster-type") String clusterType, - @JsonProperty("zts-url") String ztsUrl, - @JsonProperty("service-identity") String serviceIdentity) { - this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); - } - - @JsonProperty("signature") @Override public String signature() { return signature; } - @JsonProperty("signing-key-version") @Override public int signingKeyVersion() { return signingKeyVersion; } - @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } - @JsonProperty("provider-service") @Override public String providerService() { return providerService; } - @JsonProperty("document-version") @Override public int documentVersion() { return documentVersion; } - @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } - @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } - @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } - @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } - @JsonProperty("identity-type") @Override public String identityType() { return identityType; } - @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } - @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } - @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } - @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } - @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index 174c76f7fa9..fc0dff3b97b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -1,77 +1,57 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.api.bindings; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.DatabindContext; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; -import com.fasterxml.jackson.databind.jsontype.TypeIdResolver; -import com.fasterxml.jackson.databind.type.TypeFactory; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; - -import java.io.IOException; -import java.util.Objects; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "document-version", visible = true) -@JsonTypeIdResolver(SignedIdentityDocumentEntityTypeResolver.class) -public interface SignedIdentityDocumentEntity { - int documentVersion(); -} - -class SignedIdentityDocumentEntityTypeResolver implements TypeIdResolver { - JavaType javaType; - - @Override - public void init(JavaType javaType) { - this.javaType = javaType; - } - - @Override - public String idFromValue(Object o) { - return idFromValueAndType(o, o.getClass()); - } - - @Override - public String idFromValueAndType(Object o, Class<?> aClass) { - if (Objects.isNull(o)) { - throw new IllegalArgumentException("Cannot serialize null oject"); - } else { - if (o instanceof SignedIdentityDocumentEntity s) { - return Integer.toString(s.documentVersion()); - } else { - throw new IllegalArgumentException("Cannot serialize class: " + o.getClass()); - } - } - } - - @Override - public String idFromBaseType() { - return idFromValueAndType(null, javaType.getRawClass()); - } - - @Override - public JavaType typeFromId(DatabindContext databindContext, String s) throws IOException { - try { - int version = Integer.parseInt(s); - Class<? extends SignedIdentityDocumentEntity> cls = version <= SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION - ? LegacySignedIdentityDocumentEntity.class - : DefaultSignedIdentityDocumentEntity.class; - return TypeFactory.defaultInstance().constructSpecializedType(javaType,cls); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Unable to deserialize document with version: \"%s\"".formatted(s)); - } +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author bjorncs + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record SignedIdentityDocumentEntity( + String signature, int signingKeyVersion, String providerUniqueId, String providerService, int documentVersion, + String configServerHostname, String instanceHostname, Instant createdAt, Set<String> ipAddresses, + String identityType, String clusterType, String ztsUrl, String serviceIdentity, Map<String, Object> unknownAttributes) { + + @JsonCreator + public SignedIdentityDocumentEntity(@JsonProperty("signature") String signature, + @JsonProperty("signing-key-version") int signingKeyVersion, + @JsonProperty("provider-unique-id") String providerUniqueId, + @JsonProperty("provider-service") String providerService, + @JsonProperty("document-version") int documentVersion, + @JsonProperty("configserver-hostname") String configServerHostname, + @JsonProperty("instance-hostname") String instanceHostname, + @JsonProperty("created-at") Instant createdAt, + @JsonProperty("ip-addresses") Set<String> ipAddresses, + @JsonProperty("identity-type") String identityType, + @JsonProperty("cluster-type") String clusterType, + @JsonProperty("zts-url") String ztsUrl, + @JsonProperty("service-identity") String serviceIdentity) { + this(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity, new HashMap<>()); } - @Override - public String getDescForKnownTypeIds() { - return "Type resolver for SignedIdentityDocumentEntity"; - } - - @Override - public JsonTypeInfo.Id getMechanism() { - return JsonTypeInfo.Id.CUSTOM; - } -}
\ No newline at end of file + @JsonProperty("signature") @Override public String signature() { return signature; } + @JsonProperty("signing-key-version") @Override public int signingKeyVersion() { return signingKeyVersion; } + @JsonProperty("provider-unique-id") @Override public String providerUniqueId() { return providerUniqueId; } + @JsonProperty("provider-service") @Override public String providerService() { return providerService; } + @JsonProperty("document-version") @Override public int documentVersion() { return documentVersion; } + @JsonProperty("configserver-hostname") @Override public String configServerHostname() { return configServerHostname; } + @JsonProperty("instance-hostname") @Override public String instanceHostname() { return instanceHostname; } + @JsonProperty("created-at") @Override public Instant createdAt() { return createdAt; } + @JsonProperty("ip-addresses") @Override public Set<String> ipAddresses() { return ipAddresses; } + @JsonProperty("identity-type") @Override public String identityType() { return identityType; } + @JsonProperty("cluster-type") @Override public String clusterType() { return clusterType; } + @JsonProperty("zts-url") @Override public String ztsUrl() { return ztsUrl; } + @JsonProperty("service-identity") @Override public String serviceIdentity() { return serviceIdentity; } + @JsonAnyGetter @Override public Map<String, Object> unknownAttributes() { return unknownAttributes; } + @JsonAnySetter public void set(String name, Object value) { unknownAttributes.put(name, value); } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 1858653c9b4..cc9d3b2be65 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; @@ -75,9 +74,7 @@ class AthenzCredentialsService { } KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(); - // Use legacy version for now. - SignedIdentityDocument signedDocument = identityDocumentClient.getTenantIdentityDocument(hostname, SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION); - IdentityDocument document = signedDocument.identityDocument(); + SignedIdentityDocument document = identityDocumentClient.getTenantIdentityDocument(hostname); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( tenantIdentity, document.providerUniqueId(), @@ -90,17 +87,16 @@ class AthenzCredentialsService { ztsClient.registerInstance( configserverIdentity, tenantIdentity, - EntityBindingsMapper.toAttestationData(signedDocument), + EntityBindingsMapper.toAttestationData(document), csr); X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(keyPair.getPrivate(), certificate, signedDocument); - return new AthenzCredentials(certificate, keyPair, signedDocument); + writeCredentialsToDisk(keyPair.getPrivate(), certificate, document); + return new AthenzCredentials(certificate, keyPair, document); } } - AthenzCredentials updateCredentials(SignedIdentityDocument signedDocument, SSLContext sslContext) { + AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - IdentityDocument document = signedDocument.identityDocument(); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( tenantIdentity, document.providerUniqueId(), @@ -116,8 +112,8 @@ class AthenzCredentialsService { document.providerUniqueId().asDottedString(), csr); X509Certificate certificate = instanceIdentity.certificate(); - writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, signedDocument); - return new AthenzCredentials(certificate, newKeyPair, signedDocument); + writeCredentialsToDisk(newKeyPair.getPrivate(), certificate, document); + return new AthenzCredentials(certificate, newKeyPair, document); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index b5579cbfcd8..b9f9f3862c2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -298,7 +298,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } private X509Certificate requestRoleCertificate(AthenzRole role) { - var doc = credentials.getIdentityDocument().identityDocument(); + var doc = credentials.getIdentityDocument(); Pkcs10Csr csr = csrGenerator.generateRoleCsr( identity, role, doc.providerUniqueId(), doc.clusterType(), credentials.getKeyPair()); try (ZtsClient client = createZtsClient()) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index 48fc021dced..5b884e3dfb3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -56,16 +56,16 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { } @Override - public SignedIdentityDocument getNodeIdentityDocument(String host, int documentVersion) { - return getIdentityDocument(host, "node", documentVersion); + public SignedIdentityDocument getNodeIdentityDocument(String host) { + return getIdentityDocument(host, "node"); } @Override - public SignedIdentityDocument getTenantIdentityDocument(String host, int documentVersion) { - return getIdentityDocument(host, "tenant", documentVersion); + public SignedIdentityDocument getTenantIdentityDocument(String host) { + return getIdentityDocument(host, "tenant"); } - private SignedIdentityDocument getIdentityDocument(String host, String type, int documentVersion) { + private SignedIdentityDocument getIdentityDocument(String host, String type) { try (CloseableHttpClient client = createHttpClient(sslContextSupplier.get(), hostnameVerifier)) { URI uri = configserverUri @@ -76,7 +76,6 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { .setUri(uri) .addHeader("Connection", "close") .addHeader("Accept", "application/json") - .addParameter("documentVersion", Integer.toString(documentVersion)) .build(); try (CloseableHttpResponse response = client.execute(request)) { String responseContent = EntityUtils.toString(response.getEntity()); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java index 11b30585933..019f73fc6bf 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSigner.java @@ -4,10 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.client; import com.yahoo.security.SignatureUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; @@ -22,7 +19,7 @@ import java.util.Base64; import java.util.Set; import java.util.TreeSet; -import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; +import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -32,25 +29,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; */ public class IdentityDocumentSigner { - public String generateSignature(String identityDocumentData, PrivateKey privateKey) { - try { - Signature signer = SignatureUtils.createSigner(privateKey); - signer.initSign(privateKey); - signer.update(identityDocumentData.getBytes(UTF_8)); - byte[] signature = signer.sign(); - return Base64.getEncoder().encodeToString(signature); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } - - public String generateLegacySignature(IdentityDocument doc, PrivateKey privateKey) { - return generateSignature(doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), - doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType(), privateKey, doc.serviceIdentity()); - } - // Cluster type is ignored due to old Vespa versions not forwarding unknown fields in signed identity document - private String generateSignature(VespaUniqueInstanceId providerUniqueId, + public String generateSignature(VespaUniqueInstanceId providerUniqueId, AthenzService providerService, String configServerHostname, String instanceHostname, @@ -74,32 +54,14 @@ public class IdentityDocumentSigner { } public boolean hasValidSignature(SignedIdentityDocument doc, PublicKey publicKey) { - if (doc instanceof LegacySignedIdentityDocument signedDoc) { - return validateLegacySignature(signedDoc, publicKey); - } else if (doc instanceof DefaultSignedIdentityDocument signedDoc) { - try { - Signature signer = SignatureUtils.createVerifier(publicKey); - signer.initVerify(publicKey); - signer.update(signedDoc.data().getBytes(UTF_8)); - return signer.verify(Base64.getDecoder().decode(doc.signature())); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } else { - throw new IllegalArgumentException("Unknown identity document type: " + doc.getClass().getName()); - } - } - - private boolean validateLegacySignature(SignedIdentityDocument doc, PublicKey publicKey) { try { - IdentityDocument iddoc = doc.identityDocument(); Signature signer = SignatureUtils.createVerifier(publicKey); signer.initVerify(publicKey); writeToSigner( - signer, iddoc.providerUniqueId(), iddoc.providerService(), iddoc.configServerHostname(), - iddoc.instanceHostname(), iddoc.createdAt(), iddoc.ipAddresses(), iddoc.identityType()); - if (doc.documentVersion() >= LEGACY_DEFAULT_DOCUMENT_VERSION) { - writeToSigner(signer, iddoc.serviceIdentity()); + signer, doc.providerUniqueId(), doc.providerService(), doc.configServerHostname(), + doc.instanceHostname(), doc.createdAt(), doc.ipAddresses(), doc.identityType()); + if (doc.documentVersion() >= DEFAULT_DOCUMENT_VERSION) { + writeToSigner(signer, doc.serviceIdentity()); } return signer.verify(Base64.getDecoder().decode(doc.signature())); } catch (GeneralSecurityException e) { diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 513fb4cdbd3..2a68f6fd231 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -5,11 +5,8 @@ package com.yahoo.vespa.athenz.identityprovider.api; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -18,7 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class EntityBindingsMapperTest { @Test - public void legacy_persists_unknown_json_members() throws IOException { + public void persists_unknown_json_members() throws IOException { var originalJson = """ { @@ -39,8 +36,7 @@ class EntityBindingsMapperTest { } """; var entity = EntityBindingsMapper.fromString(originalJson); - assertInstanceOf(LegacySignedIdentityDocument.class, entity); - assertEquals(2, entity.identityDocument().unknownAttributes().size(), entity.identityDocument().unknownAttributes().toString()); + assertEquals(2, entity.unknownAttributes().size(), entity.unknownAttributes().toString()); var json = EntityBindingsMapper.toAttestationData(entity); var expectedMemberInJson = "member-in-unknown-object"; @@ -49,39 +45,4 @@ class EntityBindingsMapperTest { assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); } - @Test - public void reads_unknown_json_members() throws IOException { - var iddoc = """ - { - "provider-unique-id": "0.cluster.instance.app.tenant.us-west-1.test.node", - "provider-service": "domain.service", - "configserver-hostname": "cfg", - "instance-hostname": "host", - "created-at": 12345.0, - "ip-addresses": [], - "identity-type": "node", - "cluster-type": "admin", - "zts-url": "https://zts.url/", - "unknown-string": "string-value", - "unknown-object": { "member-in-unknown-object": 123 } - } - """; - var originalJson = - """ - { - "signature": "sig", - "signing-key-version": 0, - "document-version": 4, - "data": "%s" - } - """.formatted(Base64.getEncoder().encodeToString(iddoc.getBytes(StandardCharsets.UTF_8))); - var entity = EntityBindingsMapper.fromString(originalJson); - assertEquals(2, entity.identityDocument().unknownAttributes().size(), entity.identityDocument().unknownAttributes().toString()); - var json = EntityBindingsMapper.toAttestationData(entity); - - // For the new iddoc format the identity document should be unchanged during serialization/deserialization, - // i.e the signed identity document should be unchanged - assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); - } - }
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index acb0905700f..ff85cb79f02 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -6,13 +6,10 @@ import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.ClusterType; -import com.yahoo.vespa.athenz.identityprovider.api.DefaultSignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.LegacySignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.junit.jupiter.api.Test; import java.security.KeyPair; @@ -21,7 +18,6 @@ import java.util.Arrays; import java.util.HashSet; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; -import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -46,53 +42,32 @@ public class IdentityDocumentSignerTest { private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); @Test - void legacy_generates_and_validates_signature() { - IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); - - SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); - - assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); - } - - @Test void generates_and_validates_signature() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - String data = EntityBindingsMapper.toIdentityDocmentData(identityDocument); String signature = - signer.generateSignature(data, keyPair.getPrivate()); + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); - SignedIdentityDocument signedIdentityDocument = new DefaultSignedIdentityDocument( - signature, KEY_VERSION, DEFAULT_DOCUMENT_VERSION, data); + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); } @Test - void legacy_ignores_cluster_type_and_zts_url() { + void ignores_cluster_type_and_zts_url() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); - IdentityDocument withoutIgnoredFields = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); - String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); - var docWithoutIgnoredFields = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, withoutIgnoredFields); - var docWithIgnoredFields = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); + var docWithoutIgnoredFields = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, null, null, serviceIdentity); + var docWithIgnoredFields = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); assertTrue(signer.hasValidSignature(docWithoutIgnoredFields, keyPair.getPublic())); assertEquals(docWithIgnoredFields.signature(), docWithoutIgnoredFields.signature()); @@ -101,15 +76,16 @@ public class IdentityDocumentSignerTest { @Test void validates_signature_for_new_and_old_versions() { IdentityDocumentSigner signer = new IdentityDocumentSigner(); - IdentityDocument identityDocument = new IdentityDocument( - id, providerService, configserverHostname, - instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); String signature = - signer.generateLegacySignature(identityDocument, keyPair.getPrivate()); + signer.generateSignature(id, providerService, configserverHostname, instanceHostname, createdAt, + ipAddresses, identityType, keyPair.getPrivate(), serviceIdentity); - SignedIdentityDocument signedIdentityDocument = new LegacySignedIdentityDocument( - signature, KEY_VERSION, LEGACY_DEFAULT_DOCUMENT_VERSION, identityDocument); + SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( + signature, KEY_VERSION, id, providerService, DEFAULT_DOCUMENT_VERSION, configserverHostname, + instanceHostname, createdAt, ipAddresses, identityType, clusterType, ztsUrl, serviceIdentity); assertTrue(signer.hasValidSignature(signedIdentityDocument, keyPair.getPublic())); + } + }
\ No newline at end of file diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 0d007097fa2..1c2db273653 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -154,20 +154,20 @@ org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:3.3.1 org.eclipse.collections:eclipse-collections:11.0.0 org.eclipse.collections:eclipse-collections-api:11.0.0 -org.eclipse.jetty:jetty-alpn-client:11.0.14 -org.eclipse.jetty:jetty-alpn-java-server:11.0.14 -org.eclipse.jetty:jetty-alpn-server:11.0.14 -org.eclipse.jetty:jetty-client:11.0.14 -org.eclipse.jetty:jetty-http:11.0.14 -org.eclipse.jetty:jetty-io:11.0.14 -org.eclipse.jetty:jetty-jmx:11.0.14 -org.eclipse.jetty:jetty-security:11.0.14 -org.eclipse.jetty:jetty-server:11.0.14 -org.eclipse.jetty:jetty-servlet:11.0.14 -org.eclipse.jetty:jetty-util:11.0.14 -org.eclipse.jetty.http2:http2-common:11.0.14 -org.eclipse.jetty.http2:http2-hpack:11.0.14 -org.eclipse.jetty.http2:http2-server:11.0.14 +org.eclipse.jetty:jetty-alpn-client:11.0.15 +org.eclipse.jetty:jetty-alpn-java-server:11.0.15 +org.eclipse.jetty:jetty-alpn-server:11.0.15 +org.eclipse.jetty:jetty-client:11.0.15 +org.eclipse.jetty:jetty-http:11.0.15 +org.eclipse.jetty:jetty-io:11.0.15 +org.eclipse.jetty:jetty-jmx:11.0.15 +org.eclipse.jetty:jetty-security:11.0.15 +org.eclipse.jetty:jetty-server:11.0.15 +org.eclipse.jetty:jetty-servlet:11.0.15 +org.eclipse.jetty:jetty-util:11.0.15 +org.eclipse.jetty.http2:http2-common:11.0.15 +org.eclipse.jetty.http2:http2-hpack:11.0.15 +org.eclipse.jetty.http2:http2-server:11.0.15 org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 8ee3957af32..91365d446c1 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -31,6 +31,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT exceptions.cpp execution_profiler.cpp executor_idle_tracking.cpp + featureset.cpp file_area_freelist.cpp foregroundtaskexecutor.cpp gate.cpp diff --git a/searchlib/src/vespa/searchlib/common/featureset.cpp b/vespalib/src/vespa/vespalib/util/featureset.cpp index 5c8d4c6d9c4..6ac90461cfb 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.cpp +++ b/vespalib/src/vespa/vespalib/util/featureset.cpp @@ -2,7 +2,7 @@ #include "featureset.h" -namespace search { +namespace vespalib { FeatureSet::FeatureSet() : _names(), @@ -87,4 +87,4 @@ FeatureSet::getFeaturesByDocId(uint32_t docId) const return 0; } -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/featureset.h b/vespalib/src/vespa/vespalib/util/featureset.h index adda8a2728b..ae7a0c6932f 100644 --- a/searchlib/src/vespa/searchlib/common/featureset.h +++ b/vespalib/src/vespa/vespalib/util/featureset.h @@ -2,14 +2,13 @@ #pragma once -#include "feature.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/data/memory.h> #include <map> #include <vector> #include <memory> -namespace search { +namespace vespalib { /** * This class holds information about a set of features for a set of @@ -153,4 +152,4 @@ struct FeatureValues { std::vector<Value> values; // values.size() == names.size() * N }; -} // namespace search +} |