Merge branch 'master' into balder/less-unused-header-body-references

17 files changed, 445 insertions, 72 deletions

diff --git a/document/abi-spec.json b/document/abi-spec.json
index 9b67eeb97ee..abbf03bc228 100644
--- a/document/abi-spec.json
+++ b/document/abi-spec.json
@@ -427,6 +427,8 @@
     "methods": [
       "public void <init>(java.lang.String)",
       "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)",
+      "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)",
+      "public void <init>(java.lang.String, java.util.Set)",
       "public com.yahoo.document.DocumentType clone()",
       "public com.yahoo.document.Document createFieldValue()",
       "public java.lang.Class getValueClass()",
@@ -448,6 +450,8 @@
       "public com.yahoo.document.Field getField(int)",
       "public boolean hasField(java.lang.String)",
       "public int getFieldCount()",
+      "public java.util.Set getImportedFieldNames()",
+      "public boolean hasImportedField(java.lang.String)",
       "public com.yahoo.document.Field removeField(java.lang.String)",
       "public java.util.Collection getFields()",
       "public java.util.Set fieldSet()",
@@ -712,6 +716,8 @@
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype build()"
     ],
     "fields": [
@@ -719,7 +725,8 @@
       "public java.util.List datatype",
       "public java.util.List annotationtype",
       "public java.util.Map fieldsets",
-      "public java.util.List referencetype"
+      "public java.util.List referencetype",
+      "public java.util.List importedfield"
     ]
   },
   "com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": {
@@ -1264,6 +1271,35 @@
     ],
     "fields": []
   },
+  "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "com.yahoo.config.ConfigBuilder"
+    ],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()"
+    ],
+    "fields": []
+  },
+  "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": {
+    "superClass": "com.yahoo.config.InnerNode",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+      "public java.lang.String name()"
+    ],
+    "fields": []
+  },
   "com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
@@ -1347,7 +1383,9 @@
       "public java.util.Map fieldsets()",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)",
       "public java.util.List referencetype()",
-      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)"
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)",
+      "public java.util.List importedfield()",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)"
     ],
     "fields": []
   },
diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java
index 42ccca9f477..23559878fbb 100755
--- a/document/src/main/java/com/yahoo/document/DocumentType.java
+++ b/document/src/main/java/com/yahoo/document/DocumentType.java
@@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType {
     private StructDataType bodyType;
     private List<DocumentType> inherits = new ArrayList<>(1);
     private Map<String, Set<Field>> fieldSets = new HashMap<>();
+    private final Set<String> importedFieldNames;
 
     /**
      * Creates a new document type and registers it with the document type manager.
@@ -51,8 +52,7 @@ public class DocumentType extends StructuredDataType {
      * @param name The name of the new document type
      */
     public DocumentType(String name) {
-        this(name, new StructDataType(name + ".header"),
-                   new StructDataType(name + ".body"));
+        this(name, createHeaderStructType(name), createBodyStructType(name));
     }
 
     /**
@@ -65,9 +65,27 @@ public class DocumentType extends StructuredDataType {
      * @param bodyType   The type of the body struct
      */
     public DocumentType(String name, StructDataType headerType, StructDataType bodyType) {
+        this(name, headerType, bodyType, Collections.emptySet());
+    }
+
+    public DocumentType(String name, StructDataType headerType,
+                        StructDataType bodyType, Set<String> importedFieldNames) {
         super(name);
         this.headerType = headerType;
         this.bodyType = bodyType;
+        this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames);
+    }
+
+    public DocumentType(String name, Set<String> importedFieldNames) {
+        this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames);
+    }
+
+    private static StructDataType createHeaderStructType(String name) {
+        return new StructDataType(name + ".header");
+    }
+
+    private static StructDataType createBodyStructType(String name) {
+        return new StructDataType(name + ".body");
     }
 
     @Override
@@ -369,6 +387,14 @@ public class DocumentType extends StructuredDataType {
         return headerType.getFieldCount() + bodyType.getFieldCount();
     }
 
+    public Set<String> getImportedFieldNames() {
+        return importedFieldNames;
+    }
+
+    public boolean hasImportedField(String fieldName) {
+        return importedFieldNames.contains(fieldName);
+    }
+
     /**
      * Removes an field from the DocumentType.
      *
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 5e698e980ff..21a163aa2b9 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -10,8 +10,10 @@ import com.yahoo.log.LogLevel;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * Configures the Vespa document manager from a config id.
@@ -144,7 +146,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         for (Field field : body.getFields()) {
             field.setHeader(false);
         }
-        DocumentType type = new DocumentType(doc.name(), header, body);
+        var importedFields = doc.importedfield().stream()
+                .map(f -> f.name())
+                .collect(Collectors.toUnmodifiableSet());
+        DocumentType type = new DocumentType(doc.name(), header, body, importedFields);
         for (Object j : doc.inherits()) {
             DocumentmanagerConfig.Datatype.Documenttype.Inherits parent =
                     (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j;
diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java
index 295b1c9dd5c..134681a4028 100755
--- a/document/src/main/java/com/yahoo/document/FieldPath.java
+++ b/document/src/main/java/com/yahoo/document/FieldPath.java
@@ -10,7 +10,7 @@ import java.util.List;
  * This class represents a path into a document, that can be used to iterate through the document and extract the field
  * values you're interested in.
  *
- * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ * @author Thomas Gundersen
  */
 public class FieldPath implements Iterable<FieldPathEntry> {
 
diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
index 0cedda7c4f0..9e2759e1590 100644
--- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
+++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
@@ -7,6 +7,7 @@ import com.yahoo.document.Document;
 import com.yahoo.document.DocumentGet;
 import com.yahoo.document.DocumentPut;
 import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.FieldPath;
 import com.yahoo.document.datatypes.FieldPathIteratorHandler;
@@ -131,10 +132,37 @@ public class AttributeNode implements ExpressionNode {
         throw new IllegalStateException("Function '" + function + "' is not supported.");
     }
 
-    private static Object evaluateFieldPath(String fieldPth, Object value) {
+    private static boolean looksLikeComplexFieldPath(String path) {
+        for (int i = 0; i < path.length(); ++i) {
+            switch (path.charAt(i)) {
+                case '.':
+                case '{':
+                case '[':
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    private static boolean isSimpleImportedField(String path, DocumentType documentType) {
+        if (looksLikeComplexFieldPath(path)) {
+            return false;
+        }
+        return documentType.hasImportedField(path);
+    }
+
+    private static Object evaluateFieldPath(String fieldPathStr, Object value) {
         if (value instanceof DocumentPut) {
             final Document doc = ((DocumentPut) value).getDocument();
-            FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
+            if (isSimpleImportedField(fieldPathStr, doc.getDataType())) {
+                // Imported fields can only be meaningfully evaluated in the backend, so we
+                // explicitly treat them as if they are valid fields with missing values. This
+                // will be treated the same as if it's a normal field by the selection operators.
+                // This avoids any awkward interaction with Invalid values or having to
+                // augment the FieldPath code with knowledge of imported fields.
+                return null;
+            }
+            FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPathStr);
             IteratorHandler handler = new IteratorHandler();
             doc.iterateNested(fieldPath, 0, handler);
             if (handler.values.isEmpty()) {
diff --git a/document/src/test/document/documentmanager.importedfields.cfg b/document/src/test/document/documentmanager.importedfields.cfg
new file mode 100644
index 00000000000..765e290ae82
--- /dev/null
+++ b/document/src/test/document/documentmanager.importedfields.cfg
@@ -0,0 +1,65 @@
+datatype[7]
+datatype[0].id -1365874599
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name referenced_type.header
+datatype[0].structtype[0].version 9
+datatype[0].structtype[0].field[0]
+datatype[0].documenttype[0]
+datatype[1].id 278604398
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name referenced_type.body
+datatype[1].structtype[0].version 9
+datatype[1].structtype[0].field[0]
+datatype[1].documenttype[0]
+datatype[2].id 124647170
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name referenced_type
+datatype[2].documenttype[0].version 9
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1365874599
+datatype[2].documenttype[0].bodystruct 278604398
+datatype[3].id 12345678
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[3].referencetype[1]
+datatype[3].referencetype[0].target_type_id 124647170
+datatype[4].id 673066331
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name type_with_ref.header
+datatype[4].structtype[0].version 234
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name my_ref_field
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 12345678
+datatype[4].documenttype[0]
+datatype[5].id -176986064
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name type_with_ref.body
+datatype[5].structtype[0].version 234
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id -1293964543
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name type_with_ref
+datatype[6].documenttype[0].version 234
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct 673066331
+datatype[6].documenttype[0].bodystruct -176986064
+datatype[6].documenttype[0].importedfield[0].name "my_cool_imported_field"
+datatype[6].documenttype[0].importedfield[1].name "my_awesome_imported_field"
\ No newline at end of file
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
index cac77dba434..57b36d9758f 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -8,6 +8,7 @@ import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import org.junit.Test;
 
+import java.util.HashSet;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
@@ -555,6 +556,35 @@ search annotationsimplicitstruct {
         assertTrue(fieldRefType.getTargetType() == targetDocType);
     }
 
+    @Test
+    public void imported_fields_are_empty_if_no_fields_provided_in_config() {
+        var manager = createConfiguredManager("file:src/test/document/documentmanager.singlereference.cfg");
+        var docType = manager.getDocumentType("type_with_ref");
+
+        assertNotNull(docType.getImportedFieldNames());
+        assertEquals(docType.getImportedFieldNames().size(), 0);
+        assertFalse(docType.hasImportedField("foo"));
+    }
+
+    @Test
+    public void imported_fields_are_populated_from_config() {
+        var manager = createConfiguredManager("file:src/test/document/documentmanager.importedfields.cfg");
+        var docType = manager.getDocumentType("type_with_ref");
+
+        var expectedFields = new HashSet<String>();
+        expectedFields.add("my_cool_imported_field");
+        expectedFields.add("my_awesome_imported_field");
+        assertEquals(docType.getImportedFieldNames(), expectedFields);
+
+        assertTrue(docType.hasImportedField("my_cool_imported_field"));
+        assertTrue(docType.hasImportedField("my_awesome_imported_field"));
+        assertFalse(docType.hasImportedField("a_missing_imported_field"));
+    }
+
+    // TODO test clone(). Also fieldSets not part of clone()..!
+
+    // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this...
+
     // TODO test reference to own doc type
 
 }
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index b33d81ffdea..0212be8542e 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -8,10 +8,13 @@ import com.yahoo.document.select.parser.ParseException;
 import com.yahoo.document.select.parser.TokenMgrException;
 import com.yahoo.yolean.Exceptions;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -26,11 +29,15 @@ import static org.junit.Assert.fail;
  */
 public class DocumentSelectorTestCase {
 
+    @Rule
+    public final ExpectedException exceptionRule = ExpectedException.none();
+
     private static DocumentTypeManager manager = new DocumentTypeManager();
 
     @Before
     public void setUp() {
-        DocumentType type = new DocumentType("test");
+        var importedFields = new HashSet<>(List.of("my_imported_field"));
+        DocumentType type = new DocumentType("test", importedFields);
         type.addField("hint", DataType.INT);
         type.addField("hfloat", DataType.FLOAT);
         type.addField("hstring", DataType.STRING);
@@ -697,6 +704,27 @@ public class DocumentSelectorTestCase {
     }
 
     @Test
+    public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException {
+        var documents = createDocs();
+        assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.my_imported_field != null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.my_imported_field", documents.get(0)));
+        // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+        assertEquals(Result.INVALID, evaluate("test.my_imported_field > 0", documents.get(0)));
+    }
+
+    @Test
+    public void imported_fields_only_supported_for_simple_expressions() throws ParseException {
+        exceptionRule.expect(IllegalArgumentException.class);
+        // TODO we should probably handle this case specially and give a better exception message
+        exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test");
+
+        var documents = createDocs();
+        // Nested field access is NOT considered a simple expression.
+        evaluate("test.my_imported_field.foo", documents.get(0));
+    }
+
+    @Test
     public void testTicket1769674() {
         assertParseError("music.uri=\"junk",
                          "Lexical error at line -1, column 17.  Encountered: <EOF> after : \"\\\"junk\"");
diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp
index 79b849c5ba9..3441a336553 100644
--- a/document/src/tests/documentselectparsertest.cpp
+++ b/document/src/tests/documentselectparsertest.cpp
@@ -92,6 +92,10 @@ void DocumentSelectParserTest::SetUp()
     builder.document(-1673092522, "usergroup",
                      Struct("usergroup.header"),
                      Struct("usergroup.body"));
+    builder.document(1234567, "with_imported",
+                     Struct("with_imported.header"),
+                     Struct("with_imported.body"))
+                     .imported_field("my_imported_field");
     _repo = std::make_unique<DocumentTypeRepo>(builder.config());
 
     _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory);
@@ -103,7 +107,7 @@ Document::SP DocumentSelectParserTest::createDoc(
         uint64_t hlong)
 {
     const DocumentType* type = _repo->getDocumentType(doctype);
-    Document::SP doc(new Document(*type, DocumentId(id)));
+    auto doc = std::make_shared<Document>(*type, DocumentId(id));
     doc->setValue(doc->getField("headerval"), IntFieldValue(hint));
 
     if (hlong != 0) {
@@ -1524,4 +1528,26 @@ TEST_F(DocumentSelectParserTest, test_parse_utilities_handle_malformed_input)
     check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity());
 }
 
+TEST_F(DocumentSelectParserTest, imported_field_references_are_treated_as_valid_field_with_missing_value) {
+    const DocumentType* type = _repo->getDocumentType("with_imported");
+    ASSERT_TRUE(type != nullptr);
+    Document doc(*type, DocumentId("id::with_imported::foo"));
+
+    PARSE("with_imported.my_imported_field == null", doc, True);
+    PARSE("with_imported.my_imported_field != null", doc, False);
+    PARSE("with_imported.my_imported_field", doc, False);
+    // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+    PARSE("with_imported.my_imported_field > 0", doc, Invalid);
+}
+
+TEST_F(DocumentSelectParserTest, imported_field_references_only_support_for_simple_expressions) {
+    const DocumentType* type = _repo->getDocumentType("with_imported");
+    ASSERT_TRUE(type != nullptr);
+    Document doc(*type, DocumentId("id::with_imported::foo"));
+
+    PARSE("with_imported.my_imported_field.foo", doc, Invalid);
+    PARSE("with_imported.my_imported_field[0]", doc, Invalid);
+    PARSE("with_imported.my_imported_field{foo}", doc, Invalid);
+}
+
 } // document
diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
index b263ad75930..0bc80ebcd16 100644
--- a/document/src/tests/repo/documenttyperepo_test.cpp
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -532,6 +532,47 @@ TEST("Reference fields are resolved to correct reference type") {
     EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType());
 }
 
+TEST("Config with no imported fields has empty imported fields set in DocumentType") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name));
+    DocumentTypeRepo repo(builder.config());
+    const auto *type = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_TRUE(type->imported_field_names().empty());
+    EXPECT_FALSE(type->has_imported_field_name("foo"));
+}
+
+TEST("Configured imported field names are available in the DocumentType") {
+    const int type_2_id = doc_type_id + 1;
+    // Note: we cheat a bit by specifying imported field names in types that have no
+    // reference fields. Add to test if we add config read-time validation of this. :)
+    DocumenttypesConfigBuilderHelper builder;
+    // Type with one imported field
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+                     .imported_field("my_cool_field");
+    // Type with two imported fields
+    builder.document(type_2_id, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2))
+                     .imported_field("my_awesome_field")
+                     .imported_field("my_swag_field");
+
+    DocumentTypeRepo repo(builder.config());
+    const auto* type = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_EQUAL(1u, type->imported_field_names().size());
+    EXPECT_TRUE(type->has_imported_field_name("my_cool_field"));
+    EXPECT_FALSE(type->has_imported_field_name("my_awesome_field"));
+
+    type = repo.getDocumentType(type_2_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_EQUAL(2u, type->imported_field_names().size());
+    EXPECT_TRUE(type->has_imported_field_name("my_awesome_field"));
+    EXPECT_TRUE(type->has_imported_field_name("my_swag_field"));
+    EXPECT_FALSE(type->has_imported_field_name("my_cool_field"));
+}
+
 namespace {
 
 const TensorDataType &
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
index 1961f67d83c..092a29d9293 100644
--- a/document/src/vespa/document/config/documentmanager.def
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int
 ## Field sets
 datatype[].documenttype[].fieldsets{}.fields[] string
 
+## Imported fields (specified outside the document block in the schema)
+datatype[].documenttype[].importedfield[].name string
+
 ## Cross-document reference with ID of target document type
 datatype[].referencetype[].target_type_id int
 
diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def
index fb3fcd8b123..0f0a9e3e37c 100644
--- a/document/src/vespa/document/config/documenttypes.def
+++ b/document/src/vespa/document/config/documenttypes.def
@@ -104,3 +104,6 @@ documenttype[].referencetype[].id int
 
 ## Numeric ID of the document type instances of the reference point to.
 documenttype[].referencetype[].target_type_id int
+
+## Imported fields (specified outside the document block in the schema)
+documenttype[].importedfield[].name string
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index ffc250eacba..c2739dc1303 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -23,9 +23,10 @@ DocumentType::DocumentType() = default;
 DocumentType::DocumentType(stringref name, int32_t id)
     : StructuredDataType(name, id),
       _inheritedTypes(),
-      _ownedFields(new StructDataType(name + ".header")),
+      _ownedFields(std::make_shared<StructDataType>(name + ".header")),
       _fields(_ownedFields.get()),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
         _inheritedTypes.push_back(DataType::DOCUMENT);
@@ -36,7 +37,8 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
     : StructuredDataType(name, id),
       _inheritedTypes(),
       _fields(&fields),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
         _inheritedTypes.push_back(DataType::DOCUMENT);
@@ -46,12 +48,13 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
 DocumentType::DocumentType(stringref name)
     : StructuredDataType(name),
       _inheritedTypes(),
-      _ownedFields(new StructDataType(name + ".header")),
+      _ownedFields(std::make_shared<StructDataType>(name + ".header")),
       _fields(_ownedFields.get()),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
-        _inheritedTypes.push_back(DataType::DOCUMENT);
+        _inheritedTypes.emplace_back(DataType::DOCUMENT);
     }
 }
 
@@ -59,27 +62,28 @@ DocumentType::DocumentType(stringref name, const StructDataType& fields)
     : StructuredDataType(name),
       _inheritedTypes(),
       _fields(&fields),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
-        _inheritedTypes.push_back(DataType::DOCUMENT);
+        _inheritedTypes.emplace_back(DataType::DOCUMENT);
     }
 }
 
 DocumentType::~DocumentType() = default;
 
 DocumentType &
-DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields)
+DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields)
 {
-    _fieldSets[name] = FieldSet(name, fields);
+    _fieldSets[name] = FieldSet(name, std::move(fields));
     return *this;
 }
 
 const DocumentType::FieldSet *
 DocumentType::getFieldSet(const vespalib::string & name) const
 {
-    FieldSetMap::const_iterator it(_fieldSets.find(name));
-    return (it != _fieldSets.end()) ? & it->second : NULL;
+    auto it = _fieldSets.find(name);
+    return (it != _fieldSets.end()) ? & it->second : nullptr;
 }
 
 void
@@ -107,37 +111,34 @@ DocumentType::inherit(const DocumentType &docType) {
                 "Document type " + docType.toString() + " already inherits type "
                 + toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC);
     }
-        // If we already inherits this type, there is no point in adding it
-        // again.
+    // If we already inherits this type, there is no point in adding it again.
     if (isA(docType)) {
-            // If we already directly inherits it, complain
-        for (std::vector<const DocumentType *>::const_iterator
-                it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it)
-        {
-            if (**it == docType) {
+        // If we already directly inherits it, complain
+        for (const auto* inherited : _inheritedTypes) {
+            if (*inherited == docType) {
                 throw IllegalArgumentException(
                         "DocumentType " + getName() + " already inherits "
                         "document type " + docType.getName(), VESPA_STRLOC);
             }
         }
-            // Indirectly already inheriting it is oki, as this can happen
-            // due to inherited documents inheriting the same type.
+        // Indirectly already inheriting it is oki, as this can happen
+        // due to inherited documents inheriting the same type.
         LOG(info, "Document type %s inherits document type %s from multiple "
                   "types.", getName().c_str(), docType.getName().c_str());
         return;
     }
-        // Add non-conflicting types.
+    // Add non-conflicting types.
     Field::Set fs = docType._fields->getFieldSet();
-    for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) {
+    for (const auto* field : fs) {
         if (!_ownedFields.get()) {
             _ownedFields.reset(_fields->clone());
             _fields = _ownedFields.get();
         }
-        _ownedFields->addInheritedField(**it);
+        _ownedFields->addInheritedField(*field);
     }
     // If we inherit default document type Document.0, remove that if adding
     // another parent, as that has to also inherit Document
-    if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) {
+    if ((_inheritedTypes.size() == 1) && (*_inheritedTypes[0] == *DataType::DOCUMENT)) {
         _inheritedTypes.clear();
     }
     _inheritedTypes.push_back(&docType);
@@ -168,8 +169,7 @@ DocumentType::print(std::ostream& out, bool verbose, const std::string& indent)
     out << ")";
     if (verbose) {
         if (!_inheritedTypes.empty()) {
-            std::vector<const DocumentType *>::const_iterator it(
-                    _inheritedTypes.begin());
+            auto it = _inheritedTypes.begin();
             out << "\n" << indent << "    : ";
             (*it)->print(out, false, "");
             while (++it != _inheritedTypes.end()) {
@@ -188,17 +188,18 @@ DocumentType::operator==(const DataType& other) const
 {
     if (&other == this) return true;
     if (!DataType::operator==(other)) return false;
-    const DocumentType* o(dynamic_cast<const DocumentType*>(&other));
-    if (o == 0) return false;
+    const auto* o(dynamic_cast<const DocumentType*>(&other));
+    if (o == nullptr) return false;
     if (*_fields != *o->_fields) return false;
     if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false;
-    std::vector<const DocumentType *>::const_iterator it1(_inheritedTypes.begin());
-    std::vector<const DocumentType *>::const_iterator it2(o->_inheritedTypes.begin());
+    auto it1 = _inheritedTypes.begin();
+    auto it2 = o->_inheritedTypes.begin();
     while (it1 != _inheritedTypes.end()) {
         if (**it1 != **it2) return false;
         ++it1;
         ++it2;
     }
+    // TODO imported fields? like in the Java impl, field sets are not considered either... :I
     return true;
 }
 
@@ -228,6 +229,14 @@ DocumentType::getFieldSet() const
     return _fields->getFieldSet();
 }
 
+bool DocumentType::has_imported_field_name(const vespalib::string& name) const noexcept {
+    return (_imported_field_names.find(name) != _imported_field_names.end());
+}
+
+void DocumentType::add_imported_field_name(const vespalib::string& name) {
+    _imported_field_names.insert(name);
+}
+
 DocumentType *
 DocumentType::clone() const {
     return new DocumentType(*this);
diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h
index 7fcf5dfa237..ed6e9e66ab5 100644
--- a/document/src/vespa/document/datatype/documenttype.h
+++ b/document/src/vespa/document/datatype/documenttype.h
@@ -12,9 +12,11 @@
 #pragma once
 
 #include <vespa/document/datatype/structdatatype.h>
-
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/string.h>
 #include <vector>
 #include <map>
+#include <set>
 
 namespace document {
 
@@ -25,12 +27,17 @@ class DocumentType : public StructuredDataType {
 public:
     class FieldSet {
     public:
-        typedef std::set<vespalib::string> Fields;
-        FieldSet() : _name(), _fields() {}
-        FieldSet(const vespalib::string & name) : _name(name), _fields() {}
-        FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {}
-        const vespalib::string & getName() const { return _name; }
-        const Fields & getFields() const { return _fields; }
+        using Fields = std::set<vespalib::string>;
+        FieldSet() = default;
+        explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {}
+        FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {}
+        FieldSet(const FieldSet&) = default;
+        FieldSet& operator=(const FieldSet&) = default;
+        FieldSet(FieldSet&&) noexcept = default;
+        FieldSet& operator=(FieldSet&&) noexcept = default;
+
+        const vespalib::string & getName() const noexcept { return _name; }
+        const Fields & getFields() const noexcept { return _fields; }
         FieldSet & add(vespalib::string & field) {
             _fields.insert(field);
             return *this;
@@ -39,28 +46,31 @@ public:
         vespalib::string _name;
         Fields           _fields;
     };
-    typedef std::map<vespalib::string, FieldSet> FieldSetMap;
+    using FieldSetMap = std::map<vespalib::string, FieldSet>;
+    using ImportedFieldNames = vespalib::hash_set<vespalib::string>;
+
     std::vector<const DocumentType *> _inheritedTypes;
-    StructDataType::SP _ownedFields;
-    const StructDataType* _fields;
-    FieldSetMap _fieldSets;
+    StructDataType::SP                _ownedFields;
+    const StructDataType*             _fields;
+    FieldSetMap                       _fieldSets;
+    ImportedFieldNames                _imported_field_names;
 
 public:
-    typedef std::unique_ptr<DocumentType> UP;
-    typedef std::shared_ptr<DocumentType> SP;
+    using UP = std::unique_ptr<DocumentType>;
+    using SP = std::shared_ptr<DocumentType>;
 
     DocumentType();
     DocumentType(vespalib::stringref name, int32_t id);
     DocumentType(vespalib::stringref name, int32_t id,
                  const StructDataType& fields);
 
-    DocumentType(vespalib::stringref name);
+    explicit DocumentType(vespalib::stringref name);
     DocumentType(vespalib::stringref name,
                  const StructDataType& fields);
 
-    ~DocumentType();
+    ~DocumentType() override;
 
-    const StructDataType& getFieldsType() const { return *_fields; }
+    const StructDataType& getFieldsType() const noexcept { return *_fields; }
 
     void addField(const Field&);
 
@@ -89,9 +99,16 @@ public:
     Field::Set getFieldSet() const override;
     DocumentType* clone() const override;
 
-    DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields);
+    DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields);
     const FieldSet * getFieldSet(const vespalib::string & name) const;
 
+    const ImportedFieldNames& imported_field_names() const noexcept {
+        return _imported_field_names;
+    }
+    bool has_imported_field_name(const vespalib::string& name) const noexcept;
+    // Ideally the type would be immutable, but this is how it's built today.
+    void add_imported_field_name(const vespalib::string& name);
+
     DECLARE_IDENTIFIABLE(DocumentType);
 };
 
diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h
index 15ee0da0c79..52ed8d878a2 100644
--- a/document/src/vespa/document/repo/configbuilder.h
+++ b/document/src/vespa/document/repo/configbuilder.h
@@ -38,9 +38,9 @@ struct TypeOrId {
 };
 
 struct Struct : DatatypeConfig {
-    Struct(const vespalib::string &name) {
+    explicit Struct(vespalib::string name) {
         type = Type::STRUCT;
-        sstruct.name = name;
+        sstruct.name = std::move(name);
     }
     Struct &setCompression(Sstruct::Compression::Type t, int32_t level,
                            int32_t threshold, int32_t min_size) {
@@ -63,7 +63,7 @@ struct Struct : DatatypeConfig {
 };
 
 struct Array : DatatypeConfig {
-    Array(TypeOrId nested_type) {
+    explicit Array(TypeOrId nested_type) {
         addNestedType(nested_type);
         type = Type::ARRAY;
         array.element.id = nested_type.id;
@@ -71,7 +71,7 @@ struct Array : DatatypeConfig {
 };
 
 struct Wset : DatatypeConfig {
-    Wset(TypeOrId nested_type) {
+    explicit Wset(TypeOrId nested_type) {
         addNestedType(nested_type);
         type = Type::WSET;
         wset.key.id = nested_type.id;
@@ -94,7 +94,7 @@ struct Map : DatatypeConfig {
 };
 
 struct AnnotationRef : DatatypeConfig {
-    AnnotationRef(int32_t annotation_type_id) {
+    explicit AnnotationRef(int32_t annotation_type_id) {
         type = Type::ANNOTATIONREF;
         annotationref.annotation.id = annotation_type_id;
     }
@@ -103,7 +103,7 @@ struct AnnotationRef : DatatypeConfig {
 struct DocTypeRep {
     DocumenttypesConfig::Documenttype &doc_type;
 
-    DocTypeRep(DocumenttypesConfig::Documenttype &type)
+    explicit DocTypeRep(DocumenttypesConfig::Documenttype &type)
         : doc_type(type) {}
     DocTypeRep &inherit(int32_t id) {
         doc_type.inherits.resize(doc_type.inherits.size() + 1);
@@ -127,6 +127,12 @@ struct DocTypeRep {
         doc_type.referencetype.back().targetTypeId = target_type_id;
         return *this;
     }
+
+    DocTypeRep& imported_field(vespalib::string field_name) {
+        doc_type.importedfield.resize(doc_type.importedfield.size() + 1);
+        doc_type.importedfield.back().name = std::move(field_name);
+        return *this;
+    }
 };
 
 class DocumenttypesConfigBuilderHelper {
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index da59f527115..58dbd16beb6 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -441,10 +441,10 @@ void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, Do
     for (const auto & entry : fsv) {
         const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second);
         DocumentType::FieldSet::Fields fields;
-        for (size_t j(0); j < fs.fields.size(); j++) {
-            fields.insert(fs.fields[j]);
+        for (const auto& f : fs.fields) {
+            fields.insert(f);
         }
-        doc_type.addFieldSet(entry.first, fields);
+        doc_type.addFieldSet(entry.first, std::move(fields));
     }
 }
 
@@ -457,6 +457,14 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference
     }
 }
 
+void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldVector& imported_fields,
+                         DocumentType& doc_type)
+{
+    for (const auto& imported : imported_fields) {
+        doc_type.add_imported_field_name(imported.name);
+    }
+}
+
 void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) {
     DataTypeRepo *data_types = type_map[doc_type.id];
     inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations);
@@ -467,6 +475,7 @@ void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, Do
     setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo);
     inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type);
     addFieldSet(doc_type.fieldsets, *data_types->doc_type);
+    add_imported_fields(doc_type.importedfield, *data_types->doc_type);
 }
 
 void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) {
diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp
index 5c7820b76d7..95cf2f4e7e5 100644
--- a/document/src/vespa/document/select/valuenodes.cpp
+++ b/document/src/vespa/document/select/valuenodes.cpp
@@ -340,10 +340,34 @@ FieldValueNode::initFieldPath(const DocumentType& type) const {
     }
 }
 
+namespace {
+
+bool looks_like_complex_field_path(const vespalib::string& expr) {
+    for (const char c : expr) {
+        switch (c) {
+        case '.':
+        case '[':
+        case '{':
+            return true;
+        default: continue;
+        }
+    }
+    return false;
+}
+
+bool is_simple_imported_field(const vespalib::string& expr, const DocumentType& doc_type) {
+    if (looks_like_complex_field_path(expr)) {
+        return false;
+    }
+    return (doc_type.has_imported_field_name(expr));
+}
+
+}
+
 std::unique_ptr<Value>
 FieldValueNode::getValue(const Context& context) const
 {
-    if (context._doc == NULL) {
+    if (context._doc == nullptr) {
         return std::make_unique<InvalidValue>();
     }
 
@@ -352,7 +376,17 @@ FieldValueNode::getValue(const Context& context) const
     if (!documentTypeEqualsName(doc.getType(), _doctype)) {
         return std::make_unique<InvalidValue>();
     }
-    try{
+    // Imported fields can only be meaningfully evaluated inside Proton, so we
+    // explicitly treat them as if they are valid fields with missing values. This
+    // will be treated the same as if it's a normal field by the selection operators.
+    // This avoids any awkward interaction with Invalid values or having to
+    // augment the FieldPath code with knowledge of imported fields.
+    // When a selection is running inside Proton, it will patch FieldValueNodes for
+    // imported fields, which removes this check entirely.
+    if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+        return std::make_unique<NullValue>();
+    }
+    try {
         initFieldPath(doc.getType());
 
         IteratorHandler handler;
@@ -363,7 +397,7 @@ FieldValueNode::getValue(const Context& context) const
         } else {
             const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
 
-            if (values.size() == 0) {
+            if (values.empty()) {
                 return std::make_unique<NullValue>();
             } else {
                 return std::make_unique<ArrayValue>(handler.getValues());
@@ -399,7 +433,7 @@ FieldValueNode::print(std::ostream& out, bool verbose,
 std::unique_ptr<Value>
 FieldValueNode::traceValue(const Context &context, std::ostream& out) const
 {
-    if (context._doc == NULL) {
+    if (context._doc == nullptr) {
         return defaultTrace(getValue(context), out);
     }
     const Document &doc(*context._doc);
@@ -408,7 +442,12 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const
             << _doctype << " document, thus resolving invalid.\n";
         return std::make_unique<InvalidValue>();
     }
-    try{
+    if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+        out << "Field '" << _fieldExpression << "' refers to an imported field; "
+            << "returning NullValue to treat this as an unset field value.\n";
+        return std::make_unique<NullValue>();
+    }
+    try {
         initFieldPath(doc.getType());
 
         IteratorHandler handler;