1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema.processing;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.PositionDataType;
import com.yahoo.schema.RankProfileRegistry;
import com.yahoo.schema.Schema;
import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.GeoPos;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.vespa.documentmodel.DocumentSummary;
import com.yahoo.vespa.documentmodel.SummaryField;
import com.yahoo.vespa.documentmodel.SummaryField.Source;
import com.yahoo.vespa.documentmodel.SummaryTransform;
import com.yahoo.vespa.model.container.search.QueryProfiles;
/*
* Adjusts position summary fields by adding derived summary fields (.distance and .position) and setting summary
* transform and source.
*/
public class AdjustPositionSummaryFields extends Processor {
public AdjustPositionSummaryFields(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
super(schema, deployLogger, rankProfileRegistry, queryProfiles);
}
private boolean useV8GeoPositions = false;
@Override
public void process(boolean validate, boolean documentsOnly, ModelContext.Properties properties) {
this.useV8GeoPositions = properties.featureFlags().useV8GeoPositions();
process(validate, documentsOnly);
}
@Override
public void process(boolean validate, boolean documentsOnly) {
for (DocumentSummary summary : schema.getSummaries().values()) {
scanSummary(summary);
}
}
static String getPositionSummaryFieldName(String fieldName) {
// Only used in v7 legacy mode, remove in Vespa 9
return fieldName + ".position";
}
static String getDistanceSummaryFieldName(String fieldName) {
// Only used in v7 legacy mode, remove in Vespa 9
return fieldName + ".distance";
}
private void scanSummary(DocumentSummary summary) {
for (SummaryField summaryField : summary.getSummaryFields().values()) {
if ( ! GeoPos.isAnyPos(summaryField.getDataType())) continue;
String originalSource = summaryField.getSingleSource();
if (originalSource.indexOf('.') == -1) { // Eliminate summary fields with pos.x or pos.y as source
ImmutableSDField sourceField = schema.getField(originalSource);
if (sourceField != null) {
String zCurve = null;
if (sourceField.getDataType().equals(summaryField.getDataType())) {
zCurve = PositionDataType.getZCurveFieldName(originalSource);
} else if (sourceField.getDataType().equals(makeZCurveDataType(summaryField.getDataType())) &&
hasZCurveSuffix(originalSource)) {
zCurve = originalSource;
}
if (zCurve != null) {
if (hasPositionAttribute(zCurve)) {
Source source = new Source(zCurve);
adjustPositionField(summary, summaryField, source);
} else if (sourceField.isImportedField() || !summaryField.getName().equals(originalSource)) {
fail(summaryField, "No position attribute '" + zCurve + "'");
}
}
}
}
}
}
private void adjustPositionField(DocumentSummary summary, SummaryField summaryField, Source source) {
summaryField.setTransform(SummaryTransform.GEOPOS);
summaryField.getSources().clear();
summaryField.addSource(source);
ensureSummaryField(summary,
getPositionSummaryFieldName(summaryField.getName()),
DataType.getArray(DataType.STRING),
source,
SummaryTransform.POSITIONS);
ensureSummaryField(summary,
getDistanceSummaryFieldName(summaryField.getName()),
DataType.INT,
source,
SummaryTransform.DISTANCE);
}
private void ensureSummaryField(DocumentSummary summary, String fieldName, DataType dataType, Source source, SummaryTransform transform) {
SummaryField oldField = schema.getSummaryField(fieldName);
if (oldField == null) {
if (useV8GeoPositions) return;
SummaryField newField = new SummaryField(fieldName, dataType, transform);
newField.addSource(source);
summary.add(newField);
return;
}
if (!oldField.getDataType().equals(dataType)) {
fail(oldField, "exists with type '" + oldField.getDataType().toString() + "', should be of type '" + dataType.toString() + "'");
}
if (oldField.getTransform() != transform) {
fail(oldField, "has summary transform '" + oldField.getTransform().toString() + "', should have transform '" + transform.toString() + "'");
}
if (oldField.getSourceCount() != 1 || !oldField.getSingleSource().equals(source.getName())) {
fail(oldField, "has source '" + oldField.getSources().toString() + "', should have source '" + source + "'");
}
if (useV8GeoPositions) return;
summary.add(oldField);
}
private boolean hasPositionAttribute(String name) {
Attribute attribute = schema.getAttribute(name);
if (attribute == null) {
ImmutableSDField field = schema.getField(name);
if (field != null && field.isImportedField()) {
attribute = field.getAttribute();
}
}
return attribute != null && attribute.isPosition();
}
private static boolean hasZCurveSuffix(String name) {
String suffix = PositionDataType.getZCurveFieldName("");
return name.length() > suffix.length() && name.substring(name.length() - suffix.length()).equals(suffix);
}
private static DataType makeZCurveDataType(DataType dataType) {
return dataType instanceof ArrayDataType ? DataType.getArray(DataType.LONG) : DataType.LONG;
}
private void fail(SummaryField summaryField, String msg) {
throw newProcessException(schema.getName(), summaryField.getName(), msg);
}
}
|
