diff options
author | Arne H Juul <arnej@yahooinc.com> | 2022-02-15 14:17:12 +0000 |
---|---|---|
committer | Arne H Juul <arnej@yahooinc.com> | 2022-02-15 15:05:36 +0000 |
commit | f83e44760a72cc7d0226352b1114ffc3a8f97dee (patch) | |
tree | e0ed02df063aaaa0f3adab688b43018b774ce0c7 /document/src | |
parent | d29d335828d98e248f1cb593882d2d4f08fe702c (diff) |
add special DocumentOnly fieldset in Java as well
* should have same behavior in Java and C++
* extend unit tests to verify
* note various places where we want to change the default on Vespa 8 branch
Diffstat (limited to 'document/src')
6 files changed, 92 insertions, 2 deletions
diff --git a/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java b/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java new file mode 100644 index 00000000000..7ddfad13dfc --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/DocumentOnly.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +/** + * @author arnej27959 + */ +public class DocumentOnly implements FieldSet { + public static final String NAME = "[document]"; + @Override + public boolean contains(FieldSet o) { + return (o instanceof DocumentOnly || o instanceof DocIdOnly || o instanceof NoFields); + } + + @Override + public FieldSet clone() { + return new DocumentOnly(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java index 41a1b898867..9b6507d171a 100644 --- a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java +++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java @@ -21,13 +21,14 @@ public class FieldSetRepo { FieldSet parseSpecialValues(String name) { if (name.equals(DocIdOnly.NAME)) { return new DocIdOnly(); } + else if (name.equals(DocumentOnly.NAME)) { return (new DocumentOnly()); } else if (name.equals(AllFields.NAME)) { return (new AllFields()); } else if (name.equals(NoFields.NAME)) { return (new NoFields()); } else if (name.equals("[docid]")) { return (new DocIdOnly()); } else { throw new IllegalArgumentException( "The only special names (enclosed in '[]') allowed are " + - "id, all, none"); + "id, all, document, none"); } } @@ -101,6 +102,8 @@ public class FieldSetRepo { return NoFields.NAME; } else if (fieldSet instanceof DocIdOnly) { return DocIdOnly.NAME; + } else if (fieldSet instanceof DocumentOnly) { + return DocumentOnly.NAME; } else { throw new IllegalArgumentException("Unknown field set type " + fieldSet); } @@ -112,6 +115,18 @@ public class FieldSetRepo { * fieldset. */ public void copyFields(Document source, Document target, FieldSet fieldSet) { + if (fieldSet instanceof DocumentOnly) { + var actual = source.getDataType().fieldSet(DocumentOnly.NAME); + if (actual != null) { + for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + if (actual.contains(v.getKey())) { + target.setFieldValue(v.getKey(), v.getValue()); + } + } + return; + } + } for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) { Map.Entry<Field, FieldValue> v = i.next(); @@ -126,6 +141,21 @@ public class FieldSetRepo { */ public void stripFields(Document target, FieldSet fieldSet) { List<Field> toStrip = new ArrayList<>(); + if (fieldSet instanceof DocumentOnly) { + var actual = target.getDataType().fieldSet(DocumentOnly.NAME); + if (actual != null) { + for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + if (! actual.contains(v.getKey())) { + toStrip.add(v.getKey()); + } + } + for (Field f : toStrip) { + target.removeFieldValue(f); + } + return; + } + } for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) { Map.Entry<Field, FieldValue> v = i.next(); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java index bf9960c3ca3..871e54ca46c 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java @@ -5,6 +5,8 @@ import com.yahoo.document.datatypes.FloatFieldValue; import com.yahoo.document.datatypes.Raw; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertNotNull; @@ -61,6 +63,8 @@ public class DocumentTestCaseBase { stringField = testDocType.getField("stringattr"); minField = testDocType.getField("Minattr"); + testDocType.addFieldSets(Map.of("[document]", List.of("stringattr", "intattr"))); + docMan.registerDocumentType(testDocType); } diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java index e1e93adfb6d..b6dff63ac7b 100644 --- a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java +++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java @@ -23,6 +23,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { @Test public void testClone() throws Exception { assertTrue(new AllFields().clone() instanceof AllFields); + assertTrue(new DocumentOnly().clone() instanceof DocumentOnly); assertTrue(new NoFields().clone() instanceof NoFields); assertTrue(new DocIdOnly().clone() instanceof DocIdOnly); } @@ -32,6 +33,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { FieldSetRepo repo = new FieldSetRepo(); assertTrue(repo.parse(docMan, AllFields.NAME) instanceof AllFields); + assertTrue(repo.parse(docMan, DocumentOnly.NAME) instanceof DocumentOnly); assertTrue(repo.parse(docMan, NoFields.NAME) instanceof NoFields); assertTrue(repo.parse(docMan, DocIdOnly.NAME) instanceof DocIdOnly); @@ -72,21 +74,30 @@ public class FieldSetTestCase extends DocumentTestCaseBase { assertTrue(intAttr.contains(new DocIdOnly())); assertTrue(intAttr.contains(new NoFields())); assertFalse(intAttr.contains(new AllFields())); + assertFalse(intAttr.contains(new DocumentOnly())); assertFalse(new NoFields().contains(intAttr)); assertFalse(new NoFields().contains(new AllFields())); assertFalse(new NoFields().contains(new DocIdOnly())); + assertFalse(new NoFields().contains(new DocumentOnly())); assertTrue(new AllFields().contains(intAttr)); assertTrue(new AllFields().contains(rawAttr)); assertTrue(new AllFields().contains(new DocIdOnly())); + assertTrue(new AllFields().contains(new DocumentOnly())); assertTrue(new AllFields().contains(new NoFields())); assertTrue(new AllFields().contains(new AllFields())); assertTrue(new DocIdOnly().contains(new NoFields())); assertTrue(new DocIdOnly().contains(new DocIdOnly())); + assertFalse(new DocIdOnly().contains(new DocumentOnly())); assertFalse(new DocIdOnly().contains(intAttr)); + assertTrue(new DocumentOnly().contains(new NoFields())); + assertTrue(new DocumentOnly().contains(new DocIdOnly())); + assertTrue(new DocumentOnly().contains(new DocumentOnly())); + assertFalse(new DocumentOnly().contains(intAttr)); + assertContains("testdoc:rawattr,intattr", "testdoc:intattr"); assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr"); assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr"); @@ -124,6 +135,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { doc.removeFieldValue("rawattr"); assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, AllFields.NAME)); + assertEquals("stringattr:tjohei,intattr:50", doCopyFields(doc, DocumentOnly.NAME)); assertEquals("floatattr:3.56,byteattr:30", doCopyFields(doc, "testdoc:floatattr,byteattr")); } @@ -140,6 +152,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { doc.removeFieldValue("rawattr"); assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, AllFields.NAME)); + assertEquals("stringattr:tjohei,intattr:50", doStripFields(doc, DocumentOnly.NAME)); assertEquals("floatattr:3.56,byteattr:30", doStripFields(doc, "testdoc:floatattr,byteattr")); } @@ -150,6 +163,7 @@ public class FieldSetTestCase extends DocumentTestCaseBase { AllFields.NAME, NoFields.NAME, DocIdOnly.NAME, + DocumentOnly.NAME, "testdoc:rawattr", "testdoc:rawattr,intattr" }; diff --git a/document/src/tests/fieldsettest.cpp b/document/src/tests/fieldsettest.cpp index b4ee7713613..9f00fdd8c0d 100644 --- a/document/src/tests/fieldsettest.cpp +++ b/document/src/tests/fieldsettest.cpp @@ -28,6 +28,7 @@ TEST_F(FieldSetTest, testParsing) const DocumentTypeRepo& docRepo = testDocMan.getTypeRepo(); (void) dynamic_cast<AllFields&>(*FieldSetRepo::parse(docRepo, AllFields::NAME)); + (void) dynamic_cast<DocumentOnly&>(*FieldSetRepo::parse(docRepo, DocumentOnly::NAME)); (void) dynamic_cast<NoFields&>(*FieldSetRepo::parse(docRepo, NoFields::NAME)); (void) dynamic_cast<DocIdOnly&>(*FieldSetRepo::parse(docRepo, DocIdOnly::NAME)); @@ -73,6 +74,7 @@ TEST_F(FieldSetTest, testContains) NoFields none; AllFields all; + DocumentOnly doconly; DocIdOnly id; EXPECT_EQ(false, headerField.contains(type.getField("headerlongval"))); @@ -87,6 +89,8 @@ TEST_F(FieldSetTest, testContains) EXPECT_EQ(true, all.contains(id)); EXPECT_EQ(false, none.contains(id)); EXPECT_EQ(true, id.contains(none)); + EXPECT_EQ(true, doconly.contains(none)); + EXPECT_EQ(true, doconly.contains(id)); EXPECT_EQ(true, checkContains(repo, "testdoctype1:content,headerval", @@ -181,6 +185,9 @@ TEST_F(FieldSetTest, testCopyDocumentFields) "headerval: 5678\n" "hstringval: hello fantastic world\n"), doCopyFields(*src, repo, AllFields::NAME)); + EXPECT_EQ(std::string("headerval: 5678\n" + "hstringval: hello fantastic world\n"), + doCopyFields(*src, repo, DocumentOnly::NAME)); EXPECT_EQ(std::string("content: megafoo megabar\n" "hstringval: hello fantastic world\n"), doCopyFields(*src, repo, "testdoctype1:hstringval,content")); @@ -219,9 +226,19 @@ TEST_F(FieldSetTest, testDocumentSubsetCopy) EXPECT_EQ(doCopyFields(*src, repo, AllFields::NAME), stringifyFields(*doc)); } + { + Document::UP doc(FieldSet::createDocumentSubsetCopy(*src, DocumentOnly())); + // Test that document id and type are copied correctly. + EXPECT_TRUE(doc.get()); + EXPECT_EQ(src->getId(), doc->getId()); + EXPECT_EQ(src->getType(), doc->getType()); + EXPECT_EQ(doCopyFields(*src, repo, DocumentOnly::NAME), + stringifyFields(*doc)); + } const char* fieldSets[] = { AllFields::NAME, + DocumentOnly::NAME, NoFields::NAME, "testdoctype1:hstringval,content" }; @@ -239,6 +256,7 @@ TEST_F(FieldSetTest, testSerialize) const char* fieldSets[] = { AllFields::NAME, NoFields::NAME, + DocumentOnly::NAME, DocIdOnly::NAME, "testdoctype1:content", "testdoctype1:content,hstringval" @@ -264,6 +282,9 @@ TEST_F(FieldSetTest, testStripFields) "headerval: 5678\n" "hstringval: hello fantastic world\n"), doStripFields(*src, repo, AllFields::NAME)); + EXPECT_EQ(std::string("headerval: 5678\n" + "hstringval: hello fantastic world\n"), + doStripFields(*src, repo, DocumentOnly::NAME)); EXPECT_EQ(std::string("content: megafoo megabar\n" "hstringval: hello fantastic world\n"), doStripFields(*src, repo, "testdoctype1:hstringval,content")); diff --git a/document/src/vespa/document/base/testdocrepo.cpp b/document/src/vespa/document/base/testdocrepo.cpp index fed14eef792..c1930de8551 100644 --- a/document/src/vespa/document/base/testdocrepo.cpp +++ b/document/src/vespa/document/base/testdocrepo.cpp @@ -27,6 +27,7 @@ DocumenttypesConfig TestDocRepo::getDefaultConfig() { const int mystruct_id = -2092985851; const int structarray_id = 759956026; config_builder::DocumenttypesConfigBuilderHelper builder; + ::config::StringVector documentfields = { "headerval", "hstringval", "title" }; builder.document(type1_id, "testdoctype1", Struct("testdoctype1.header") .addField("headerval", DataType::T_INT) @@ -55,7 +56,9 @@ DocumenttypesConfig TestDocRepo::getDefaultConfig() { .addTensorField("sparse_tensor", "tensor(x{})") .addTensorField("sparse_xy_tensor", "tensor(x{},y{})") .addTensorField("sparse_float_tensor", "tensor<float>(x{})") - .addTensorField("dense_tensor", "tensor(x[2])")); + .addTensorField("dense_tensor", "tensor(x[2])")) + .doc_type.fieldsets["[document]"].fields.swap(documentfields); + builder.document(type2_id, "testdoctype2", Struct("testdoctype2.header") .addField("onlyinchild", DataType::T_INT), |