diff options
98 files changed, 1213 insertions, 386 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c71589efaca..378439a4fbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,15 @@ # @author Arnstein Ressem cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +include(functions.cmake) +list(APPEND CMAKE_MODULE_PATH "$ENV{HOME}/share/cmake/Modules" "/opt/vespa-deps/share/cmake/Modules") +include(default_build_settings.cmake) +vespa_detect_build_platform() +message("-- Vespa build platform is ${VESPA_OS_DISTRO} ${VESPA_OS_DISTRO_VERSION}") +vespa_use_default_cxx_compiler() +vespa_use_default_java_home() +vespa_use_default_build_settings() + project(vespa CXX C) # allows import of project in CLion on OSX @@ -14,7 +23,6 @@ endif() # TODO: Move this to where it's actually needed find_package(JNI REQUIRED) -include(functions.cmake) include(build_settings.cmake) # Enable CTest unit testing diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index a117d283146..7dc3496adc9 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -45,6 +45,7 @@ import java.util.logging.Logger; * @author bjorncs */ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvider { + private static final String CERTIFICATE_ALIAS = "athenz"; private static final Duration EXPIRATION_MARGIN = Duration.ofHours(6); private static final Path VESPA_SIA_DIRECTORY = Paths.get(Defaults.getDefaults().underVespaHome("var/vespa/sia")); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index d1675f189c0..d943cf27f9c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.jrt.ListenFailedException; import com.yahoo.log.LogLevel; import com.yahoo.vdslib.distribution.ConfiguredNode; @@ -30,8 +31,6 @@ import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import java.io.FileNotFoundException; -import java.io.PrintWriter; -import java.io.StringWriter; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; @@ -548,10 +547,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } catch (Exception e) { responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; message = "Internal Server Error"; - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw, true); - e.printStackTrace(pw); - hiddenMessage = sw.getBuffer().toString(); + hiddenMessage = ExceptionUtils.getStackTraceAsString(e);; log.log(LogLevel.DEBUG, "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index b736bae82c8..6e4c0bc7eaa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -1,14 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status.statuspage; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.log.LogLevel; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -186,11 +185,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { response.setResponseCode(StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR); StringBuilder content = new StringBuilder(); response.writeHtmlHeader(content, "Internal Server Error"); - try (StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw, true)) { - e.printStackTrace(pw); - response.writeHtmlFooter(content, sw.getBuffer().toString()); - } + response.writeHtmlFooter(content, ExceptionUtils.getStackTraceAsString(e)); response.writeContent(content.toString()); } if (response == null) { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index e28c5eac0bb..1cdb54a743c 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -68,6 +68,7 @@ public class Xml { static DocumentBuilder getPreprocessDocumentBuilder() throws ParserConfigurationException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); // XXE prevention factory.setNamespaceAware(true); factory.setXIncludeAware(false); factory.setValidating(false); diff --git a/config-model/pom.xml b/config-model/pom.xml index 180a2e40921..25b733985f5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -46,10 +46,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> @@ -297,14 +293,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_http_service</artifactId> <version>${project.version}</version> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java new file mode 100644 index 00000000000..d8f02b2a9c3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +/** + * @author bjorncs + */ +class Utils { + + private Utils() {} + + // Separate class since javacc does not accept Java code using lambdas + static int count(String str, char ch) { + return (int) str.chars().filter(c -> c == ch).count(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 0d99c698aca..8db4ec432c5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -23,6 +23,7 @@ public class ImplicitSummaryFields extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { + if (docsum.getInherited() != null) continue; // Implicit fields are added to inheriting summaries through their parent addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 8dc5356026b..3c6aa881af4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.Field; - import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -16,6 +14,7 @@ import java.util.List; public class DocumentSummary extends FieldView { private boolean fromDisk = false; + private DocumentSummary inherited; /** * Creates a DocumentSummary with the given name. @@ -44,13 +43,21 @@ public class DocumentSummary extends FieldView { } public SummaryField getSummaryField(String name) { + var parent = getInherited(); + if (parent != null) { + return parent.getSummaryField(name); + } return (SummaryField) get(name); } public Collection<SummaryField> getSummaryFields() { - ArrayList<SummaryField> fields = new ArrayList<>(getFields().size()); - for(Field f : getFields()) { - fields.add((SummaryField) f); + var fields = new ArrayList<SummaryField>(getFields().size()); + var parent = getInherited(); + if (parent != null) { + fields.addAll(parent.getSummaryFields()); + } + for (var field : getFields()) { + fields.add((SummaryField) field); } return fields; } @@ -80,8 +87,19 @@ public class DocumentSummary extends FieldView { } } + /** Sets the parent of this. Both summaries must be present in the same search definition */ + public void setInherited(DocumentSummary inherited) { + this.inherited = inherited; + } + + /** Returns the parent of this, or null if none is inherited */ + public DocumentSummary getInherited() { + return inherited; + } + public String toString() { - return "document summary '" + getName() + "'"; + return "document summary '" + getName() + "'" + + (inherited == null ? "" : " inheriting from '" + inherited.getName() + "'"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index bedf8057b41..08956b272f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -306,9 +306,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config * types in the model. * - * @param clazz The type of config - * @param configId The config id - * @return A config instance of the given type + * @param clazz the type of config + * @param configId the config id + * @return a config instance of the given type */ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) { try { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 30f08d13772..f9762ce58fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -10,7 +10,6 @@ import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; -import org.apache.commons.lang.StringUtils; import java.util.ArrayList; import java.util.Collection; @@ -67,7 +66,7 @@ public class ComplexAttributeFieldsValidator extends Validator { } private static String toString(ImmutableSDField field) { - return field.getName() + " (" + StringUtils.join(getStructFieldAttributes(field.getStructFields()), ", ") + ")"; + return field.getName() + " (" + String.join(", ", getStructFieldAttributes(field.getStructFields())) + ")"; } private static boolean hasStructFieldAttributes(Collection<? extends ImmutableSDField> structFields) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index 3a97bf4b876..d94cd57357d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -4,14 +4,13 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.ChangesRequiringRestart; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.RestartConfigs; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.utils.internal.ReflectionUtil; -import org.apache.commons.lang3.ClassUtils; import java.time.Instant; import java.util.Arrays; @@ -95,7 +94,7 @@ public class ConfigValueChangeValidator implements ChangeValidator { * NOTE: Only the super classes that are subclass of Service are inspected. */ private static Stream<Class<? extends ConfigInstance>> getConfigInstancesFromServiceAnnotations(Class<? extends Service> serviceClass) { - List<Class<?>> classHierarchy = ClassUtils.getAllSuperclasses(serviceClass); + List<Class<?>> classHierarchy = ReflectionUtil.getAllSuperclasses(serviceClass); classHierarchy.add(serviceClass); return classHierarchy.stream() .filter(Service.class::isAssignableFrom) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 55979023119..1263b9bed6e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -118,21 +118,33 @@ public class ConvertedModel { RankProfile rankProfile, QueryProfileRegistry queryProfileRegistry, ImportedMlModel importedModel) { - ModelStore modelStore = new ModelStore(rankProfile.applicationPackage(), modelName); - return new ConvertedModel(modelName, - modelDescription, - convertAndStore(importedModel, rankProfile, queryProfileRegistry, modelStore), - Optional.of(importedModel)); + try { + ModelStore modelStore = new ModelStore(rankProfile.applicationPackage(), modelName); + return new ConvertedModel(modelName, + modelDescription, + convertAndStore(importedModel, rankProfile, queryProfileRegistry, modelStore), + Optional.of(importedModel)); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("In " + rankProfile + ": Could not create model '" + modelName + + " (" + modelDescription + ")", e); + } } public static ConvertedModel fromStore(ModelName modelName, String modelDescription, RankProfile rankProfile) { - ModelStore modelStore = new ModelStore(rankProfile.applicationPackage(), modelName); - return new ConvertedModel(modelName, - modelDescription, - convertStored(modelStore, rankProfile), - Optional.empty()); + try { + ModelStore modelStore = new ModelStore(rankProfile.applicationPackage(), modelName); + return new ConvertedModel(modelName, + modelDescription, + convertStored(modelStore, rankProfile), + Optional.empty()); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("In " + rankProfile + ": Could not create model '" + modelName + + " (" + modelDescription + ")", e); + } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java index ef021872efb..ab46bb287b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java @@ -9,6 +9,8 @@ import com.yahoo.vespa.model.ConfigProducer; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -80,6 +82,14 @@ public final class ReflectionUtil { } } + public static List<Class<?>> getAllSuperclasses(Class<?> cls) { + var result = new ArrayList<Class<?>>(); + for(Class<?> superClass = cls.getSuperclass(); superClass != null; superClass = superClass.getSuperclass()) { + result.add(superClass); + } + return result; + } + private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) { try { String defName = ConfigInstance.getDefName(configInstClass); diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index e560d78a116..52665ff56a9 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -63,7 +63,6 @@ import java.util.List; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.logging.Level; -import org.apache.commons.lang.StringUtils; /** * A search definition parser @@ -1706,6 +1705,7 @@ Object documentSummary(Search search) : { ( <DOCUMENTSUMMARY> name = identifierWithDash() { search.addSummary(summary = new DocumentSummary(name)); } + [inheritsDocumentSummary(summary, search)] lbrace() ( <FROMDISK> { summary.setFromDisk(true); } | @@ -1718,6 +1718,23 @@ Object documentSummary(Search search) : } /** + * This rule consumes an inherits statement of a document summary. + * + * @param documentSummary The document summary to modify. + * @param search The search object documentSummary is being added to. + */ +void inheritsDocumentSummary(DocumentSummary documentSummary, Search search) : +{ + String name; +} +{ + <INHERITS> name = identifierWithDash() + { + documentSummary.setInherited(search.getSummaries().get(name)); + } +} + +/** * Consumes a single document-summary item. * * @param summary The document summary to modify. @@ -2429,7 +2446,8 @@ void importField(Search search) : <IMPORT> <FIELD> fieldRefSpec = identifier() <AS> aliasFieldName = identifier() lbrace() <RBRACE> { - if (StringUtils.countMatches(fieldRefSpec, ".") != 1) { + long nDots = Utils.count(fieldRefSpec, '.'); + if (nDots != 1) { throw new IllegalArgumentException("Illegal field reference spec '" + fieldRefSpec + "': Does not include a single '.'"); } int indexOfDot = fieldRefSpec.indexOf('.'); diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 4634e120a3a..2e0a207d249 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -59,7 +59,7 @@ attribute[].arity 8 attribute[].lowerbound -9223372036854775808 attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 -attribute[].tensortype "tensor(x[10],y[10])" +attribute[].tensortype "tensor(x[10],y[20])" attribute[].imported false attribute[].name "f5" attribute[].datatype TENSOR diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 68bd394c9d6..72fae572b76 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -35,7 +35,7 @@ documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x{})" documenttype[].datatype[].sstruct.field[].name "f4" documenttype[].datatype[].sstruct.field[].id 1224191509 documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[10])" +documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[20])" documenttype[].datatype[].sstruct.field[].name "f5" documenttype[].datatype[].sstruct.field[].id 329055840 documenttype[].datatype[].sstruct.field[].datatype 21 diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index 7970e05b790..29dc39b01ce 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -4,7 +4,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "unranked" @@ -21,7 +21,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile1" @@ -34,27 +34,27 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile2" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f2" rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile3" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -64,7 +64,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile4" @@ -77,7 +77,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile5" @@ -90,14 +90,14 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile6" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -107,7 +107,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile7" @@ -128,7 +128,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile8" @@ -143,7 +143,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" rankprofile[].name "profile9" @@ -158,7 +158,7 @@ rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor<float>(x[10])" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index a7248fe3200..aa33684a979 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -11,7 +11,7 @@ search tensor { field f3 type tensor<double>(x{}) { indexing: attribute | summary } - field f4 type tensor(x[10],y[10]) { + field f4 type tensor(x[10],y[20]) { indexing: attribute | summary } field f5 type tensor<float>(x[10]) { @@ -33,7 +33,7 @@ search tensor { rank-profile profile2 { first-phase { - expression: sum(matmul(attribute(f4), diag(x[10],y[10],z[3]), x)) + expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x)) } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 686fb2173da..91599e6f607 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -2,13 +2,19 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; +import com.yahoo.vespa.objects.FieldBase; import org.junit.Test; +import java.util.Collection; +import java.util.List; import java.util.logging.Level; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests summary validation @@ -130,4 +136,119 @@ public class SummaryTestCase { assertTrue(logger.entries.isEmpty()); } + @Test + public void testInheritance() throws Exception { + String sd = + "search music {\n" + + "\n" + + " document music {\n" + + " field title type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " \n" + + " field artist type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " \n" + + " field album type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " }\n" + + " \n" + + " document-summary title {\n" + + " summary title type string {\n" + + " source: title\n" + + " }\n" + + " }\n" + + " document-summary title_artist inherits title {\n" + + " summary artist type string {\n" + + " source: artist\n" + + " }\n" + + " }\n" + + " document-summary everything inherits title_artist {\n" + + " summary album type string {\n" + + " source: album\n" + + " }\n" + + " }\n" + + "\n" + + "}"; + var logger = new DeployLoggerStub(); + var search = SearchBuilder.createFromString(sd, logger).getSearch(); + assertEquals(List.of(), logger.entries); + + var titleField = "title"; + var artistField = "artist"; + var albumField = "album"; + var titleSummary = search.getSummary(titleField); + var titleArtistSummary = search.getSummary(titleField + "_" + artistField); + var everythingSummary = search.getSummary("everything"); + + var implicitFields = List.of("rankfeatures", "summaryfeatures"); + var tests = List.of( + new TestValue(titleSummary, null, List.of(List.of(titleField), implicitFields)), + new TestValue(titleArtistSummary, titleSummary, List.of(List.of(titleField), implicitFields, List.of(artistField))), + new TestValue(everythingSummary, titleArtistSummary, List.of(List.of(titleField), implicitFields, List.of(artistField, albumField))) + ); + tests.forEach(testValue -> { + var actualFields = testValue.summary.getSummaryFields().stream() + .map(FieldBase::getName) + .collect(Collectors.toList()); + assertEquals(testValue.summary.getName() + (testValue.parent == null ? " does not inherit anything" : " inherits " + testValue.parent.getName()), + testValue.parent, + testValue.summary.getInherited()); + assertEquals("Summary " + testValue.summary.getName() + " has expected fields", testValue.fields, actualFields); + }); + } + + @Test + public void testRedeclaringInheritedFieldFails() throws Exception { + String sd = + "search music {\n" + + "\n" + + " document music {\n" + + " field title type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " field title_short type string {\n" + + " indexing: summary | attribute | index\n" + + " }\n" + + " }\n" + + " \n" + + " document-summary title {\n" + + " summary title type string {\n" + + " source: title\n" + + " }\n" + + " }\n" + + " document-summary title2 inherits title {\n" + + " summary title type string {\n" + + " source: title_short\n" + + " }\n" + + " }\n" + + " \n" + + "}"; + var logger = new DeployLoggerStub(); + try { + SearchBuilder.createFromString(sd, logger); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For search 'music', summary class 'title2', summary field 'title': Can not use " + + "source 'title_short' for this summary field, an equally named field in summary class 'title' " + + "uses a different source: 'title'.", e.getMessage()); + } + } + + private static class TestValue { + + private final DocumentSummary summary; + private final DocumentSummary parent; + private final List<String> fields; + + public TestValue(DocumentSummary summary, DocumentSummary parent, List<List<String>> fields) { + this.summary = summary; + this.parent = parent; + this.fields = fields.stream().flatMap(Collection::stream).collect(Collectors.toList());; + } + + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index a651bbb7772..d2a840d1fbc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -5,13 +5,14 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.search.grouping.GroupingValidator; -import org.apache.commons.io.IOUtils; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -50,7 +51,7 @@ public class BundleInstantiationSpecificationBuilderTest { xml += " bundle=\"" + explicitBundle + "\""; } xml += " />"; - InputStream xmlStream = IOUtils.toInputStream(xml); + InputStream xmlStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java index e5d6e149c70..fa38b415cd6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.content.ContentSearch; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -52,7 +52,7 @@ public class DomContentSearchBuilderTest { return DomContentSearchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java index d9db6234f1c..107e8968c1a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.SearchCoverage; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -71,7 +71,7 @@ public class DomSearchCoverageBuilderTest { return DomSearchCoverageBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java index 46b5b630f9e..0f909b7a8eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilderTest.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.TuningDispatch; -import org.apache.commons.io.input.CharSequenceInputStream; import org.junit.Test; import javax.xml.parsers.DocumentBuilderFactory; +import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertEquals; @@ -93,7 +93,7 @@ public class DomTuningDispatchBuilderTest { return DomTuningDispatchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(new CharSequenceInputStream(xml, StandardCharsets.UTF_8)) + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) .getDocumentElement())); } } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java index 0888004ef02..4da27e8c346 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -25,6 +25,7 @@ import java.util.Stack; * @author Ulf Lilleengen */ public class ConfigVerification { + private final static int port = 19071; private final static String prefix = "http://"; diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java index e4cedfafeea..9473a05332f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java @@ -7,5 +7,7 @@ package com.yahoo.vespa.config.buildergen; * @author Ulf Lilleengen */ public interface ConfigCompiler { + CompiledBuilder compile(ConfigDefinitionClass defClass); + } diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java index 2fc3cd7aa19..2af19d1c6f2 100644 --- a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java @@ -16,9 +16,9 @@ import java.util.Locale; * {@link CompiledBuilder}. * * @author Ulf Lilleengen - * @since 5.2 */ public class LazyConfigCompiler implements ConfigCompiler { + private final File outputDirectory; private final ClassLoader classLoader; private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); diff --git a/configserver/pom.xml b/configserver/pom.xml index b39686c31d9..1e242988327 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -206,11 +206,6 @@ <artifactId>commons-compress</artifactId> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>2.4</version> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> <scope>test</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 401b6974596..13ef19f5f5d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -17,7 +17,6 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; -import org.apache.commons.io.IOUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -249,7 +248,7 @@ public class ZooKeeperClient { } ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream inputStream = file.createInputStream()) { - IOUtils.copy(inputStream, baos); + inputStream.transferTo(baos); baos.flush(); configCurator.putData(zkPath.append(file.getPath().getName()).getAbsolute(), baos.toByteArray()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java index 595575531d8..07d27ffd5de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java @@ -3,13 +3,12 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; -import org.apache.commons.io.IOUtils; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpResponse.Status.OK; /** * Represents a response for a request to read contents of a file. @@ -28,7 +27,7 @@ public class SessionContentReadResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { try (InputStream inputStream = file.createInputStream()) { - IOUtils.copyLarge(inputStream, outputStream, new byte[1]); + inputStream.transferTo(outputStream); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java index 707ba6e361e..d027d08eda5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import org.apache.commons.io.IOUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,7 +28,7 @@ public class StaticResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { - IOUtils.copy(body, outputStream); + body.transferTo(outputStream); body.close(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 01c6631111e..a106f52e4d1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.google.common.io.Files; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -24,6 +23,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertNotNull; @@ -145,8 +145,8 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { } private File createTestApp() throws IOException { - File testApp = Files.createTempDir(); - FileUtils.copyDirectory(new File("src/test/apps/content"), testApp); + File testApp = Files.createTempDirectory("session-content-handler-test-app").toFile(); + IOUtils.copyDirectory(new File("src/test/apps/content"), testApp); return testApp; } diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index c82cfd000d9..53c17257992 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -51,10 +51,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <scope>test</scope> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index 72eaf129453..b943c03f48f 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -2,7 +2,6 @@ package com.yahoo.container.logging; import com.yahoo.collections.ListMap; -import org.apache.commons.lang.builder.ReflectionToStringBuilder; import javax.security.auth.x500.X500Principal; import java.net.InetAddress; @@ -456,9 +455,34 @@ public class AccessLogEntry { @Override public String toString() { synchronized (monitor) { - return new ReflectionToStringBuilder(this) - .setExcludeFieldNames(FIELDS_EXCLUDED_FROM_TOSTRING) - .toString(); + return "AccessLogEntry{" + + "ipV4AddressInDotDecimalNotation='" + ipV4AddressInDotDecimalNotation + '\'' + + ", timeStampMillis=" + timeStampMillis + + ", durationBetweenRequestResponseMillis=" + durationBetweenRequestResponseMillis + + ", numBytesReturned=" + numBytesReturned + + ", remoteAddress='" + remoteAddress + '\'' + + ", remotePort=" + remotePort + + ", peerAddress='" + peerAddress + '\'' + + ", peerPort=" + peerPort + + ", profile='" + profile + '\'' + + ", errorMessage='" + errorMessage + '\'' + + ", fileName='" + fileName + '\'' + + ", userAgent='" + userAgent + '\'' + + ", referer='" + referer + '\'' + + ", user='" + user + '\'' + + ", hitCounts=" + hitCounts + + ", httpMethod='" + httpMethod + '\'' + + ", httpVersion='" + httpVersion + '\'' + + ", hostString='" + hostString + '\'' + + ", statusCode=" + statusCode + + ", scheme='" + scheme + '\'' + + ", localPort=" + localPort + + ", principal=" + principal + + ", sslPrincipal=" + sslPrincipal + + ", rawPath='" + rawPath + '\'' + + ", rawQuery='" + rawQuery + '\'' + + ", keyValues=" + keyValues + + '}'; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java index 0dc9a33e3f1..0dd2f4692d9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java @@ -16,9 +16,10 @@ import java.io.PrintStream; * <li>debug true|false</li> * </ol> * - * @author <a href="mailto:vlarsen@yahoo-inc.com">Vidar Larsen</a> + * @author Vidar Larsen */ class UnicodePropertyDump { + public static void main(String[] arg) { int start = 0; int end = 0xffff; diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java index e26bf153eb3..5f1cfccf549 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java @@ -94,6 +94,7 @@ class FederationResult { public Optional<Result> getIfAvailable(long timeout) { if (availableResult.isPresent()) return availableResult; availableResult = futureResult.getIfAvailable(timeout, TimeUnit.MILLISECONDS); + availableResult.ifPresent(result -> target.modifyTargetResult(result)); return availableResult; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index ec1836fd07b..c5573dc8fee 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -226,12 +226,15 @@ public class FederationSearcher extends ForkingSearcher { if (timeout <= 0) return Optional.empty(); Execution newExecution = new Execution(target.getChain(), execution.context()); + Result result; if (strictSearchchain) { query.resetTimeout(); - return Optional.of(newExecution.search(createFederationQuery(query, query, Window.from(query), timeout, target))); + result = newExecution.search(createFederationQuery(query, query, Window.from(query), timeout, target)); } else { - return Optional.of(newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target))); + result = newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target)); } + target.modifyTargetResult(result); + return Optional.of(result); } private FederationResult search(Query query, Execution execution, Collection<Target> targets) { @@ -558,7 +561,6 @@ public class FederationSearcher extends ForkingSearcher { } private void mergeResult(Query query, Target target, Result mergedResults, Result result) { - target.modifyTargetResult(result); ComponentId searchChainId = target.getId(); Chain<Searcher> searchChain = target.getChain(); diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index 30a3121902c..b73ef1298ee 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -153,13 +153,12 @@ public class RewriterUtils { * * @param query Query object from the searcher * @param paramName parameter to be retrieved - * * @return parameter value or null if not found */ public static String getUserParam(Query query, String paramName) { log(utilsLogger, query, "Retrieving user param value: " + paramName); - if(paramName==null) { + if (paramName == null) { error(utilsLogger, query, "Parameter name is null"); return null; } @@ -172,8 +171,7 @@ public class RewriterUtils { } /** - * Retrieve metadata passed by previous rewriter - * from query properties + * Retrieve metadata passed by previous rewriter from query properties * Initialize values if this is the first rewriter * * @param query Query object from the searcher @@ -183,10 +181,10 @@ public class RewriterUtils { log(utilsLogger, query, "Retrieving metadata passed by previous rewriter"); @SuppressWarnings("unchecked") - HashMap<String, Object> rewriteMeta = (HashMap<String, Object>) query - .properties().get(RewriterConstants.REWRITE_META); + HashMap<String, Object> rewriteMeta = + (HashMap<String, Object>)query.properties().get(RewriterConstants.REWRITE_META); - if(rewriteMeta==null) { + if (rewriteMeta == null) { log(utilsLogger, query, "No metadata available from previous rewriter"); rewriteMeta = new HashMap<>(); rewriteMeta.put(RewriterConstants.REWRITTEN, false); @@ -326,9 +324,9 @@ public class RewriterUtils { * @param msg Error message */ public static void error(Logger logger, Query query, String msg) { - if(query!=null) { + if (query != null) query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); - } logger.severe(logger.getName() + ": " + msg); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java index 603757e099d..ee38b2c9516 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -143,11 +142,6 @@ public class RoutingPolicies { // Remove active load balancers and irrelevant zones from candidates removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || !policy.zone().equals(loadBalancers.zone)); - if (!removalCandidates.isEmpty()) { - LOGGER.log(LogLevel.WARNING, "Removing " + removalCandidates + ". Active load balancers " + - activeLoadBalancers); - } - for (var policy : removalCandidates) { var dnsName = policy.endpointIn(controller.system()).dnsName(); controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); diff --git a/default_build_settings.cmake b/default_build_settings.cmake new file mode 100644 index 00000000000..deb9707ed3c --- /dev/null +++ b/default_build_settings.cmake @@ -0,0 +1,317 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +include(VespaExtendedDefaultBuildSettings OPTIONAL) + +function(setup_vespa_default_build_settings_rhel_6_10) + message("-- Setting up default build settings for rhel 6.10") +endfunction() + +function(setup_vespa_default_build_settings_rhel_7_7) + message("-- Setting up default build settings for rhel 7.7") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_rhel_8_1) + message("-- Setting up default build settings for rhel 8.1") + set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_centos_7) + message("-- Setting up default build settings for centos 7") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_centos_8) + message("-- Setting up default build settings for centos 8") + set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_darwin) + message("-- Setting up default build settings for darwin") + if(VESPA_COMPILER_VARIANT STREQUAL "gcc") + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) + elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") + set(DEFAULT_LLVM_INCLUDE_DIRECTORY "/usr/local/opt/llvm/include") + set(DEFAULT_LLVM_LINK_DIRECTORY "/usr/local/opt/llvm/lib") + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) + elseif(VESPA_COMPILER_VARIANT STREQUAL "appleclang") + set(DEFAULT_LLVM_INCLUDE_DIRECTORY "/usr/local/opt/llvm/include") + set(DEFAULT_LLVM_LINK_DIRECTORY "/usr/local/opt/llvm/lib") + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) + endif() + set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" PARENT_SCOPE) + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib") + if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY) + list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}") + endif() + list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include") + if(DEFINED DEFAULT_LLVM_INCLUDE_DIRECTORY) + list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_LLVM_INCLUDE_DIRECTORY}") + endif() + list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "/usr/local/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_fedora_29) + message("-- Setting up default build settings for fedora 29") + set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_fedora_30) + message("-- Setting up default build settings for fedora 30") + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_fedora_31) + message("-- Setting up default build settings for fedora 31") + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_fedora_32) + message("-- Setting up default build settings for fedora 32") + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) +endfunction() + +function(setup_vespa_default_build_settings_ubuntu_18_10) + message("-- Setting up default build settings for ubuntu 18.10") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-6.0/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-6.0/include" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE) +endfunction() + +function(vespa_use_default_build_settings) + if (DEFINED CMAKE_INSTALL_PREFIX AND DEFINED CMAKE_PREFIX_PATH AND + DEFINED VESPA_LLVM_VERSION AND + DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND + DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH) + return() + endif() + set(VESPA_DEPS "/opt/vespa-deps") + unset(DEFAULT_VESPA_LLVM_VERSION) + unset(DEFAULT_CMAKE_PREFIX_PATH) + unset(DEFAULT_EXTRA_LINK_DIRECTORY) + unset(DEFAULT_EXTRA_INCLUDE_DIRECTORY) + unset(DEFAULT_VESPA_USER) + if(NOT DEFINED VESPA_UNPRIVILEGED) + message("-- Setting VESPA_UNPRIVILEGED to yes") + set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE) + set(VESPA_UNPRIVILEGED "yes") + endif() + if(VESPA_UNPRIVILEGED STREQUAL "no") + set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa") + set(DEFAULT_VESPA_USER "vespa") + if(COMMAND vespa_use_specific_install_prefix) + vespa_use_specific_install_prefix() + endif() + if(COMMAND vespa_use_specific_vespa_user) + vespa_use_specific_vespa_user() + endif() + else() + set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa") + set(DEFAULT_VESPA_USER "$ENV{USER}") + endif() + if(APPLE) + if(VESPA_COMPILER_VARIANT STREQUAL "clang") + set(VESPA_DEPS "/opt/vespa-deps-clang") + elseif(VESPA_COMPILER_VARIANT STREQUAL "appleclang") + set(VESPA_DEPS "/opt/vespa-deps-appleclang") + endif() + endif() + if(COMMAND vespa_use_specific_vespa_deps) + vespa_use_specific_vespa_deps() + endif() + if(COMMAND vespa_use_specific_compiler_rpath) + vespa_use_specific_compiler_rpath() + endif() + if(COMMAND vespa_use_specific_llvm_version) + vespa_use_specific_llvm_version() + endif() + if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") + setup_vespa_default_build_settings_rhel_6_10() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.7") + setup_vespa_default_build_settings_rhel_7_7() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + setup_vespa_default_build_settings_rhel_8_1() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 7") + setup_vespa_default_build_settings_centos_7() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") + setup_vespa_default_build_settings_centos_8() + elseif(VESPA_OS_DISTRO STREQUAL "darwin") + setup_vespa_default_build_settings_darwin() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 29") + setup_vespa_default_build_settings_fedora_29() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 30") + setup_vespa_default_build_settings_fedora_30() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 31") + setup_vespa_default_build_settings_fedora_31() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32") + setup_vespa_default_build_settings_fedora_32() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 18.10") + setup_vespa_default_build_settings_ubuntu_18_10() + else() + message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") + endif() + if(NOT DEFINED VESPA_LLVM_VERSION AND NOT DEFINED DEFAULT_VESPA_LLVM_VERSION) + message(FATAL_ERROR "-- Unkonwn default llvm version") + endif() + if(NOT DEFINED DEFAULT_CMAKE_PREFIX_PATH) + set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}") + endif() + if(NOT DEFINED DEFAULT_EXTRA_LINK_DIRECTORY) + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64") + endif() + if(NOT DEFINED DEFAULT_EXTRA_INCLUDE_DIRECTORY) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include") + endif() + if(DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) + message("-- DEFAULT_CMAKE_INSTALL_PREFIX is ${DEFAULT_CMAKE_INSTALL_PREFIX}") + endif() + if(DEFINED DEFAULT_CMAKE_PREFIX_PATH) + message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") + endif() + if(DEFINED DEFAULT_EXTRA_LINK_DIRECTORY) + message("-- DEFAULT_EXTRA_LINK_DIRECTORY is ${DEFAULT_EXTRA_LINK_DIRECTORY}") + endif() + if(DEFINED DEFAULT_EXTRA_INCLUDE_DIRECTORY) + message("-- DEFAULT_EXTRA_INCLUDE_DIRECTORY is ${DEFAULT_EXTRA_INCLUDE_DIRECTORY}") + endif() + if(DEFINED DEFAULT_VESPA_LLVM_VERSION) + message("-- DEFAULT_VESPA_LLVM_VERSION is ${DEFAULT_VESPA_LLVM_VERSION}") + endif() + if(DEFINED DEFAULT_VESPA_USER) + message("-- DEFAULT_VESPA_USER is ${DEFAULT_VESPA_USER}") + endif() + if(NOT DEFINED CMAKE_INSTALL_PREFIX AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX) + message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}") + set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" PARENT_SCOPE) + set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}") + endif() + if(NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED DEFAULT_CMAKE_PREFIX_PATH) + message("-- Setting CMAKE_PREFIX_PATH to ${DEFAULT_CMAKE_PREFIX_PATH}") + set(CMAKE_PREFIX_PATH "${DEFAULT_CMAKE_PREFIX_PATH}" PARENT_SCOPE) + endif() + if(NOT DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED DEFAULT_EXTRA_INCLUDE_DIRECTORY) + message("-- Setting EXTRA_INCLUDE_DIRECTORY to ${DEFAULT_EXTRA_INCLUDE_DIRECTORY}") + set(EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) + endif() + if(NOT DEFINED EXTRA_LINK_DIRECTORY AND DEFINED DEFAULT_EXTRA_LINK_DIRECTORY) + message("-- Setting EXTRA_LINK_DIRECTORY to ${DEFAULT_EXTRA_LINK_DIRECTORY}") + set(EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) + set(EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}") + endif() + if(NOT DEFINED CMAKE_INSTALL_RPATH) + if(NOT "${VESPA_COMPILER_RPATH}" STREQUAL "${CMAKE_INSTALL_PREFIX}/lib64") + list(APPEND CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib64") + endif() + if(DEFINED EXTRA_LINK_DIRECTORY) + list(APPEND CMAKE_INSTALL_RPATH "${EXTRA_LINK_DIRECTORY}") + endif() + if(DEFINED VESPA_COMPILER_RPATH) + list(APPEND CMAKE_INSTALL_RPATH "${VESPA_COMPILER_RPATH}") + endif() + message("-- Setting CMAKE_INSTALL_RPATH to ${CMAKE_INSTALL_RPATH}") + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" PARENT_SCOPE) + endif() + if(NOT DEFINED CMAKE_BUILD_RPATH AND DEFINED VESPA_COMPILER_RPATH) + if(DEFINED EXTRA_LINK_DIRECTORY) + list(APPEND CMAKE_BUILD_RPATH "${EXTRA_LINK_DIRECTORY}") + endif() + list(APPEND CMAKE_BUILD_RPATH "${VESPA_COMPILER_RPATH}") + if(DEFINED CMAKE_BUILD_RPATH) + message("-- Setting CMAKE_BUILD_RPATH to ${CMAKE_BUILD_RPATH}") + set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" PARENT_SCOPE) + endif() + endif() + if(NOT DEFINED VESPA_LLVM_VERSION AND DEFINED DEFAULT_VESPA_LLVM_VERSION) + message("-- Setting VESPA_LLVM_VERSION to ${DEFAULT_VESPA_LLVM_VERSION}") + set(VESPA_LLVM_VERSION "${DEFAULT_VESPA_LLVM_VERSION}" PARENT_SCOPE) + endif() + if(NOT DEFINED VESPA_USER AND DEFINED DEFAULT_VESPA_USER) + message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}") + set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE) + endif() +endfunction() + +function(vespa_use_default_cxx_compiler) + if (DEFINED CMAKE_C_COMPILER AND DEFINED CMAKE_CXX_COMPILER) + return() + endif() + unset(DEFAULT_CMAKE_C_COMPILER) + unset(DEFAULT_CMAKE_CXX_COMPILER) + if(NOT DEFINED VESPA_COMPILER_VARIANT) + set(VESPA_COMPILER_VARIANT "gcc") + set(VESPA_COMPILER_VARIANT "gcc" PARENT_SCOPE) + endif() + if(VESPA_COMPILER_VARIANT STREQUAL "gcc") + if(APPLE) + set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-9" PARENT_SCOPE) + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-9" PARENT_SCOPE) + endif() + elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") + if(APPLE) + set(DEFAULT_CMAKE_C_COMPILER, "/usr/local/opt/llvm/bin/clang" PARENT_SCOPE) + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++" PARENT_SCOPE) + elseif(VESPA_OS_DISTRO STREQUAL "fedora") + set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang") + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++") + else() + message(FATAL_ERROR "-- clang compiler variant not supported for ${VESPA_OS_DISTRO_COMBINED}") + endif() + elseif(VESPA_COMPILER_VARIANT STREQUAL "appleclang") + if(APPLE) + set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang" PARENT_SCOPE) + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++" PARENT_SCOPE) + else() + message(FATAL_ERROR "-- appleclang compiler variant not supported for ${VESPA_OS_DISTRO_COMBINED}") + endif() + else() + message(FATAL_ERROR "-- unknown compiler variant ${VESPA_COMPILER_VARIANT}") + endif() + if(COMMAND vespa_use_specific_cxx_compiler) + vespa_use_specific_cxx_compiler() + endif() + if(DEFINED DEFAULT_CMAKE_C_COMPILER) + message("-- DEFAULT_CMAKE_C_COMPILER is ${DEFAULT_CMAKE_C_COMPILER}") + endif() + if(DEFINED DEFAULT_CMAKE_CXX_COMPILER) + message("-- DEFAULT_CMAKE_CXX_COMPILER is ${DEFAULT_CMAKE_CXX_COMPILER}") + endif() + if(DEFINED DEFAULT_EXTRA_LINK_DIRECTORY) + message("-- DEFAULT_EXTRA_LINK_DIRECTORY is ${DEFAULT_EXTRA_LINK_DIRECTORY}") + endif() + if(NOT DEFINED CMAKE_C_COMPILER AND DEFINED DEFAULT_CMAKE_C_COMPILER) + message("-- Setting CMAKE_C_COMPILER to ${DEFAULT_CMAKE_C_COMPILER}") + set(CMAKE_C_COMPILER "${DEFAULT_CMAKE_C_COMPILER}" PARENT_SCOPE) + endif() + if(NOT DEFINED CMAKE_CXX_COMPILER AND DEFINED DEFAULT_CMAKE_CXX_COMPILER) + message("-- Setting CMAKE_CXX_COMPILER to ${DEFAULT_CMAKE_CXX_COMPILER}") + set(CMAKE_CXX_COMPILER "${DEFAULT_CMAKE_CXX_COMPILER}" PARENT_SCOPE) + endif() +endfunction() + +function(vespa_use_default_java_home) + if (DEFINED JAVA_HOME) + return() + endif() + set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-11-openjdk") + if(APPLE) + execute_process(COMMAND "/usr/libexec/java_home" OUTPUT_VARIABLE DEFAULT_JAVA_HOME) + string(STRIP "${DEFAULT_JAVA_HOME}" DEFAULT_JAVA_HOME) + elseif(VESPA_OS_DISTRO STREQUAL "ubuntu") + set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" PARENT_SCOPE) + endif() + if(COMMAND vespa_use_specific_java_home) + vespa_use_specific_java_home() + endif() + message("-- DEFAULT_JAVA_HOME is ${DEFAULT_JAVA_HOME}") + if(NOT DEFINED JAVA_HOME AND DEFINED DEFAULT_JAVA_HOME) + message("-- Setting JAVA_HOME to ${DEFAULT_JAVA_HOME}") + set(JAVA_HOME "${DEFAULT_JAVA_HOME}" PARENT_SCOPE) + endif() +endfunction() diff --git a/dist/vespa.spec b/dist/vespa.spec index de5bc4613d2..9daeac2a1c1 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -210,6 +210,7 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ + -DVESPA_UNPRIVILEGED=no \ . make %{_smp_mflags} diff --git a/document/pom.xml b/document/pom.xml index 19f1a9470fc..1696d525f6f 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -27,10 +27,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java index f7bb5aecf0b..4a9434d43bc 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -9,8 +9,8 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.json.TokenBuffer; import com.yahoo.document.update.ValueUpdate; -import org.apache.commons.codec.binary.Base64; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -86,7 +86,7 @@ public class SingleValueReader { public static FieldValue readAtomic(String field, DataType expectedType) { if (expectedType.equals(DataType.RAW)) { - return expectedType.createFieldValue(new Base64().decode(field)); + return expectedType.createFieldValue(Base64.getMimeDecoder().decode(field)); } else if (expectedType.equals(PositionDataType.INSTANCE)) { return PositionDataType.fromString(field); } else if (expectedType instanceof ReferenceDataType) { diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index a0e7f81ea73..85bc4d032ff 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -5,8 +5,8 @@ import com.yahoo.document.Document; import com.yahoo.document.Field; import com.yahoo.document.datatypes.*; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -84,7 +84,7 @@ public class XmlSerializationHelper { public static void printRawXml(Raw r, XmlStream xml) { xml.addAttribute("binaryencoding", "base64"); - xml.addContent(new Base64(0).encodeToString(r.getByteBuffer().array())); + xml.addContent(Base64.getEncoder().encodeToString(r.getByteBuffer().array())); } public static void printStringXml(StringFieldValue s, XmlStream xml) { @@ -92,7 +92,7 @@ public class XmlSerializationHelper { if (containsNonPrintableCharactersString(content)) { byte[] bytecontent = Utf8.toBytes(content); xml.addAttribute("binaryencoding", "base64"); - xml.addContent(new Base64(0).encodeToString(bytecontent)); + xml.addContent(Base64.getEncoder().encodeToString(bytecontent)); } else { xml.addContent(content); } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java index 737371f2375..02682fa943f 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java @@ -16,12 +16,12 @@ import com.yahoo.document.serialization.DeserializationException; import com.yahoo.document.serialization.FieldReader; import com.yahoo.text.Utf8; import com.yahoo.vespa.objects.FieldBase; -import org.apache.commons.codec.binary.Base64; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import java.io.InputStream; import java.math.BigInteger; +import java.util.Base64; import java.util.Optional; /** @@ -242,7 +242,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { } private void assignPositionFieldFromStringIfNonEmpty(Struct value, String elementText, boolean base64) { - String str = base64 ? Utf8.toString(new Base64().decode(elementText)) : elementText; + String str = base64 ? Utf8.toString(Base64.getMimeDecoder().decode(elementText)) : elementText; str = str.trim(); if (str.isEmpty()) { return; @@ -409,7 +409,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { public void read(FieldBase field, Raw value) { try { if (isBase64EncodedElement(reader)) { - value.assign(new Base64().decode(reader.getElementText())); + value.assign(Base64.getMimeDecoder().decode(reader.getElementText())); } else { value.assign(reader.getElementText().getBytes()); } @@ -422,7 +422,7 @@ public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { public void read(FieldBase field, PredicateFieldValue value) { try { if (isBase64EncodedElement(reader)) { - value.assign(Predicate.fromBinary(new Base64().decode(reader.getElementText()))); + value.assign(Predicate.fromBinary(Base64.getMimeDecoder().decode(reader.getElementText()))); } else { value.assign(Predicate.fromString(reader.getElementText())); } diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java index 29621426a85..a3db1b06abe 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java @@ -13,13 +13,13 @@ import com.yahoo.document.predicate.FeatureSet; import com.yahoo.document.predicate.Predicate; import com.yahoo.document.serialization.DeserializationException; import com.yahoo.tensor.TensorType; -import org.apache.commons.codec.binary.Base64; import org.junit.Test; import javax.xml.stream.XMLStreamReader; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -161,7 +161,7 @@ public class VespaXmlFieldReaderTestCase { assertRead(predicate, "<document id='id:ns:my_type::' type='my_type'>" + " <my_predicate binaryencoding='base64'>" + - Base64.encodeBase64String(BinaryFormat.encode(predicate)) + + Base64.getMimeEncoder().encodeToString(BinaryFormat.encode(predicate)) + " </my_predicate>" + "</document>"); } diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index e98a56a0148..9e429757982 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -164,20 +164,23 @@ void op_tensor_peek(State &state, uint64_t param) { TensorSpec::Address addr; size_t child_cnt = 0; for (auto pos = self.spec().rbegin(); pos != self.spec().rend(); ++pos) { - if (std::holds_alternative<TensorSpec::Label>(pos->second)) { - addr.emplace(pos->first, std::get<TensorSpec::Label>(pos->second)); - } else { - assert(std::holds_alternative<TensorFunction::Child>(pos->second)); - double index = round(state.peek(child_cnt++).as_double()); - size_t dim_idx = self.param_type().dimension_index(pos->first); - assert(dim_idx != ValueType::Dimension::npos); - const auto ¶m_dim = self.param_type().dimensions()[dim_idx]; - if (param_dim.is_mapped()) { - addr.emplace(pos->first, vespalib::make_string("%ld", int64_t(index))); - } else { - addr.emplace(pos->first, size_t(index)); - } - } + std::visit(vespalib::overload + { + [&](const TensorSpec::Label &label) { + addr.emplace(pos->first, label); + }, + [&](const TensorFunction::Child &) { + double index = round(state.peek(child_cnt++).as_double()); + size_t dim_idx = self.param_type().dimension_index(pos->first); + assert(dim_idx != ValueType::Dimension::npos); + const auto ¶m_dim = self.param_type().dimensions()[dim_idx]; + if (param_dim.is_mapped()) { + addr.emplace(pos->first, vespalib::make_string("%ld", int64_t(index))); + } else { + addr.emplace(pos->first, size_t(index)); + } + } + }, pos->second); } TensorSpec spec = state.engine.to_spec(state.peek(child_cnt++)); const Value &result = self.result_type().is_double() @@ -364,9 +367,13 @@ Peek::push_children(std::vector<Child::CREF> &children) const { children.emplace_back(_param); for (const auto &dim: _spec) { - if (std::holds_alternative<Child>(dim.second)) { - children.emplace_back(std::get<Child>(dim.second)); - } + std::visit(vespalib::overload + { + [&](const Child &child) { + children.emplace_back(child); + }, + [](const TensorSpec::Label &){} + }, dim.second); } } @@ -381,17 +388,19 @@ Peek::visit_children(vespalib::ObjectVisitor &visitor) const { ::visit(visitor, "param", _param.get()); for (const auto &dim: _spec) { - if (std::holds_alternative<TensorSpec::Label>(dim.second)) { - const auto &label = std::get<TensorSpec::Label>(dim.second); - if (label.is_mapped()) { - ::visit(visitor, dim.first, label.name); - } else { - ::visit(visitor, dim.first, label.index); - } - } else { - assert(std::holds_alternative<Child>(dim.second)); - ::visit(visitor, dim.first, std::get<Child>(dim.second).get()); - } + std::visit(vespalib::overload + { + [&](const TensorSpec::Label &label) { + if (label.is_mapped()) { + ::visit(visitor, dim.first, label.name); + } else { + ::visit(visitor, dim.first, label.index); + } + }, + [&](const Child &child) { + ::visit(visitor, dim.first, child.get()); + } + }, dim.second); } } diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index c95ffd17bbe..b019ab64e18 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/overload.h> #include "tensor_spec.h" #include "lazy_params.h" #include "value_type.h" @@ -320,12 +321,15 @@ public: : Node(result_type_in), _param(param), _spec() { for (const auto &dim: spec) { - if (std::holds_alternative<TensorSpec::Label>(dim.second)) { - _spec.emplace(dim.first, std::get<TensorSpec::Label>(dim.second)); - } else { - assert(std::holds_alternative<Node::CREF>(dim.second)); - _spec.emplace(dim.first, std::get<Node::CREF>(dim.second).get()); - } + std::visit(vespalib::overload + { + [&](const TensorSpec::Label &label) { + _spec.emplace(dim.first, label); + }, + [&](const Node::CREF &ref) { + _spec.emplace(dim.first, ref.get()); + } + }, dim.second); } } const std::map<vespalib::string, MyLabel> &spec() const { return _spec; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp index f9d15a377e9..2920ca26234 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp @@ -2,6 +2,7 @@ #include "dense_tensor_peek_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/overload.h> #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/tensor/tensor.h> @@ -87,14 +88,17 @@ DenseTensorPeekFunction::optimize(const eval::TensorFunction &expr, Stash &stash for (auto dim = peek_type.dimensions().rbegin(); dim != peek_type.dimensions().rend(); ++dim) { auto dim_spec = peek->spec().find(dim->name); assert(dim_spec != peek->spec().end()); - if (std::holds_alternative<TensorSpec::Label>(dim_spec->second)) { - const auto &label = std::get<TensorSpec::Label>(dim_spec->second); - assert(label.is_indexed()); - spec.emplace_back(label.index, dim->size); - } else { - assert(std::holds_alternative<TensorFunction::Child>(dim_spec->second)); - spec.emplace_back(-1, dim->size); - } + + std::visit(vespalib::overload + { + [&](const TensorSpec::Label &label) { + assert(label.is_indexed()); + spec.emplace_back(label.index, dim->size); + }, + [&](const TensorFunction::Child &) { + spec.emplace_back(-1, dim->size); + } + }, dim_spec->second); } return stash.create<DenseTensorPeekFunction>(peek->copy_children(), spec); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 14086c2bf7a..532f7696893 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,6 +13,7 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * Definitions of feature flags. @@ -183,6 +184,12 @@ public class Flags { "Whether to disable CM3.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag MIGRATE_AWS_TARGET_GROUP_HEALTH_CHECKS = defineFeatureFlag( + "migrate-aws-health-checks", false, + "Migrate target group health checks to https", + "Takes effect on config server restart", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fsa/src/main/java/com/yahoo/fsa/FSA.java b/fsa/src/main/java/com/yahoo/fsa/FSA.java index f0fc27a15c5..5a4a0e4475f 100644 --- a/fsa/src/main/java/com/yahoo/fsa/FSA.java +++ b/fsa/src/main/java/com/yahoo/fsa/FSA.java @@ -83,7 +83,7 @@ public class FSA implements Closeable { public void delta(String string){ ByteBuffer buf = fsa.encode(string); Maps m = fsa.map(); - while(state >0 && buf.position()<buf.limit()){ + while (state >0 && buf.position()<buf.limit()){ delta(m, buf.get()); } } @@ -106,7 +106,7 @@ public class FSA implements Closeable { /** Jumps ahead by a word - if this is not the first word, it must be preceeded by space. */ public void deltaWord(String string){ - if (state!=fsa.start()) { + if (state != fsa.start()) { delta((byte)' '); } delta(string); diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java index 97453d97c6d..ab2ec75d7f3 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segment.java @@ -4,39 +4,38 @@ package com.yahoo.fsa.segmenter; /** * Class encapsulation of a segment. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segment { - int _beg; - int _end; - int _conn; + final int begin; + final int end; + final int conn; - public Segment(int b, int e, int c) - { - _beg = b; - _end = e; - _conn = c; + public Segment(int b, int e, int c) { + begin = b; + end = e; + conn = c; } public int beg() { - return _beg; + return begin; } public int end() { - return _end; + return end; } public int len() { - return _end-_beg; + return end - begin; } public int conn() { - return _conn; + return conn; } } diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java index 64c20e38b55..505085c46cf 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java @@ -9,58 +9,56 @@ import com.yahoo.fsa.FSA; /** * API for accessing the Segmenter automata. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segmenter { - private FSA _fsa; + private final FSA fsa; public Segmenter(FSA fsa) { - _fsa = fsa; + this.fsa = fsa; } public Segmenter(String filename) { - _fsa = new FSA(filename,"utf-8"); + fsa = new FSA(filename, "utf-8"); } public Segmenter(String filename, String charsetname) { - _fsa = new FSA(filename,charsetname); + fsa = new FSA(filename, charsetname); } - public boolean isOk() - { - return _fsa.isOk(); + public boolean isOk() { + return fsa.isOk(); } - public Segments segment(String input) - { + public Segments segment(String input) { String[] tokens = input.split("\\s"); return segment(tokens); } private class Detector { - FSA.State _state; - int _index; - public Detector(FSA.State s, int i) - { - _state = s; - _index = i; + final FSA.State state; + final int index; + + public Detector(FSA.State s, int i) { + state = s; + index = i; } public FSA.State state() { - return _state; + return state; } public int index() { - return _index; + return index; } + } - public Segments segment(String[] tokens) - { + public Segments segment(String[] tokens) { Segments segments = new Segments(tokens); LinkedList detectors = new LinkedList(); @@ -68,7 +66,7 @@ public class Segmenter { while(i<tokens.length){ - detectors.add(new Detector(_fsa.getState(),i)); + detectors.add(new Detector(fsa.getState(), i)); ListIterator det_it = detectors.listIterator(); while(det_it.hasNext()){ @@ -88,50 +86,5 @@ public class Segmenter { return segments; } - //// test //// - public static void main(String[] args) { - String fsafile = "/home/gv/fsa/automata/segments.fsa"; - - Segmenter segmenter = new Segmenter(fsafile); - - System.out.println("Loading segmenter FSA file "+fsafile+": "+segmenter.isOk()); - - for(int a=0;a<1||a<args.length;a++){ - - String query; - if(a==args.length){ - query = "times square head"; - } - else { - query = args[a]; - } - System.out.println("processing query \""+query+"\""); - - Segments segments = segmenter.segment(query); - System.out.println("all segments:"); - for(int i=0; i<segments.size();i++){ - System.out.println(" "+i+": \""+segments.sgm(i)+"\","+segments.conn(i)); - } - - Segments best; - - best = segments.segmentation(Segments.SEGMENTATION_WEIGHTED); - System.out.print("best segments (weighted): "); - for(int i=0; i<best.size();i++){ - System.out.print("("+best.sgm(i)+")"); - } - System.out.println(); - - best = segments.segmentation(Segments.SEGMENTATION_RIGHTMOST_LONGEST); - System.out.print("best segments (rightmost_longest):"); - for(int i=0; i<best.size();i++){ - System.out.print("("+best.sgm(i)+")"); - } - System.out.println(); - - } - - } - } diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java index 6d7ee7be1a6..e01afa91cd5 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java @@ -6,7 +6,7 @@ import java.util.LinkedList; /** * Contains the segmentation() method. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class Segments extends LinkedList { diff --git a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java index f29dc38f2ce..cc3ff2e79ac 100644 --- a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java +++ b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java @@ -25,13 +25,13 @@ import com.yahoo.fsa.MetaData; * predicted topics for a term. Each topic has an attached weight and * a term vector (topicSegments). * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> - **/ + * @author Peter Boros + */ public class TopicPredictor extends MetaData { private static final String packageName = "com.yahoo.fsa.topicpredictor"; - private FSA fsa = null; + private final FSA fsa; public TopicPredictor(String fsafile, String datfile){ this(fsafile, datfile, "utf-8"); @@ -154,27 +154,4 @@ public class TopicPredictor extends MetaData { return getStringArrayEntry(user(0) + topicId, 2); } - - //// test //// - public static void main(String[] args) { - String segment = "new york"; - if (args.length >= 1) { - segment = args[0]; - } - - String fsafile = "/home/gv/fsa/automata/dmozPred_2.fsa"; - String datfile = "/home/gv/fsa/automata/dmozPred_2.dat"; - - TopicPredictor predictor = new TopicPredictor(fsafile, datfile); - - List predictedTopics = predictor.getPredictedTopics(segment, 25); - Iterator i = predictedTopics.iterator(); - while (i.hasNext()) { - PredictedTopic topic = (PredictedTopic) i.next(); - System.out.println("\n topic=" + topic.getTopic()); - System.out.println(" weight=" + topic.getWeight()); - System.out.println(" vector=" + topic.getVector()); - } - } - } diff --git a/functions.cmake b/functions.cmake index 3212ea04a65..cc8f1c5b038 100644 --- a/functions.cmake +++ b/functions.cmake @@ -590,3 +590,26 @@ function(add_extra_projects) endif() endfunction() +function(vespa_detect_build_platform) + if(EXISTS /etc/os-release) + file(STRINGS /etc/os-release OS_DISTRO REGEX "^ID=") + string(REGEX REPLACE "ID=\"?([^\"]+)\"?" "\\1" OS_DISTRO ${OS_DISTRO}) + file(STRINGS /etc/os-release OS_DISTRO_VERSION REGEX "^VERSION_ID=") + string(REGEX REPLACE "VERSION_ID=\"?([^\"]+)\"?" "\\1" OS_DISTRO_VERSION ${OS_DISTRO_VERSION}) + elseif(EXISTS /etc/redhat-release) + set(OS_DISTRO "rhel") + file(STRINGS "/etc/redhat-release" OS_DISTRO_VERSION) + string(REGEX REPLACE ".* release ([0-9.]+) .*" "\\1" OS_DISTRO_VERSION ${OS_DISTRO_VERSION}) + elseif(APPLE) + set(OS_DISTRO "darwin") + set(OS_DISTRO_VERSION ${CMAKE_SYSTEM_VERSION}) + endif() + if(OS_DISTRO) + set(VESPA_OS_DISTRO ${OS_DISTRO} PARENT_SCOPE) + set(VESPA_OS_DISTRO_VERSION ${OS_DISTRO_VERSION} PARENT_SCOPE) + string(CONCAT OS_DISTRO_COMBINED ${OS_DISTRO} " " ${OS_DISTRO_VERSION}) + set(VESPA_OS_DISTRO_COMBINED ${OS_DISTRO_COMBINED} PARENT_SCOPE) + else() + message(FATAL_ERROR "-- Could not determine vespa build platform") + endif() +endfunction() diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 6e890fd2c2d..5a2608b63be 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -24,6 +24,7 @@ import java.util.logging.Logger; * @author gjoranv */ public class ExportPackages { + private static final Logger log = Logger.getLogger(ExportPackages.class.getName()); public static final String PROPERTIES_FILE = "/exportPackages.properties"; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java index 4d3bb4a280a..006a282e1e0 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java @@ -33,4 +33,5 @@ class JDiscSslContextFactory extends SslContextFactory.Server { Objects.toString(getTrustStoreProvider(), getKeyStoreProvider()), trustStorePassword); } + } diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 31d7543304a..5a4a79995d8 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -133,9 +133,9 @@ public class Mirror implements IMirror { * and end with '**' to match the rest of the name. * Note that this isn't quite globbing, as there is no backtracking. * - * @return true if the name matches the pattern * @param name the name * @param pattern the pattern + * @return true if the name matches the pattern */ static boolean match(char [] name, char [] pattern) { int ni = 0; diff --git a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java index 7803767e4d0..6c36e8f9604 100644 --- a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java +++ b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java @@ -28,10 +28,10 @@ import java.util.ArrayList; public class RpcInvoker { private Value getArgument(Request request, String parameter) { - if (parameter.length()<=1 || parameter.charAt(1)!=':') + if (parameter.length() <= 1 || parameter.charAt(1) != ':') return new StringValue(parameter); - String value=parameter.substring(2); + String value = parameter.substring(2); switch (parameter.charAt(0)) { case 'b': return new Int8Value(Byte.parseByte(value)); @@ -53,9 +53,9 @@ public class RpcInvoker { "There is no jrt type identified by '" + parameter.charAt(0) + "'"); } - protected Request createRequest(String method,List<String> arguments) { - Request request=new Request(method); - if (arguments!=null) { + protected Request createRequest(String method, List<String> arguments) { + Request request = new Request(method); + if (arguments != null) { for (String argument : arguments) request.parameters().add(getArgument(request,argument)); } @@ -69,37 +69,37 @@ public class RpcInvoker { * @param method the name of the method to invoke * @param arguments the argument to the method, or null or an empty list if there are no arguments */ - public void invoke(String connectspec,String method, List<String> arguments) { - Supervisor supervisor=null; - Target target=null; + public void invoke(String connectspec, String method, List<String> arguments) { + Supervisor supervisor = null; + Target target = null; try { - if (connectspec.indexOf('/')<0) - connectspec="tcp/" + connectspec; + if (connectspec.indexOf('/') < 0) + connectspec = "tcp/" + connectspec; - supervisor=new Supervisor(new Transport()); + supervisor = new Supervisor(new Transport()); target = supervisor.connect(new Spec(connectspec)); - Request request=createRequest(method,arguments); + Request request = createRequest(method,arguments); target.invokeSync(request,10.0); if (request.isError()) { System.err.println("error(" + request.errorCode() + "): " + request.errorMessage()); return; } - Values returned=request.returnValues(); - for (int i=0; i<returned.size(); i++) { + Values returned = request.returnValues(); + for (int i = 0; i < returned.size(); i++) { System.out.println(returned.get(i)); } } finally { - if (target!=null) + if (target != null) target.close(); - if (supervisor!=null) + if (supervisor != null) supervisor.transport().shutdown().join(); } } public static void main(String[] args) { - if (args.length<1) { + if (args.length < 1) { System.err.println("usage: invoke [-h <connectspec>] <method> [arguments]"); System.err.println(" Connectspec: This is on the form hostname:port, or tcp/hostname:port"); System.err.println(" if omitted, localhost:8086 is used"); @@ -107,14 +107,14 @@ public class RpcInvoker { System.err.println(" supported types: {'b','h','i','l','f','d','s'}"); System.exit(0); } - List<String> arguments=new ArrayList<String>(Arrays.asList(args)); - String connectSpec="localhost:8086"; - if ("-h".equals(arguments.get(0)) && arguments.size()>=3) { + List<String> arguments = new ArrayList<String>(Arrays.asList(args)); + String connectSpec = "localhost:8086"; + if ("-h".equals(arguments.get(0)) && arguments.size() >= 3) { arguments.remove(0); // Consume -h - connectSpec=arguments.remove(0); + connectSpec = arguments.remove(0); } - String method=arguments.remove(0); - new RpcInvoker().invoke(connectSpec,method,arguments); + String method = arguments.remove(0); + new RpcInvoker().invoke(connectSpec, method, arguments); } } diff --git a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java index 26021492df2..4d801cd6229 100644 --- a/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java +++ b/logserver/src/main/java/com/yahoo/plugin/SystemPropertyConfig.java @@ -11,19 +11,18 @@ package com.yahoo.plugin; * @author Stig Bakken */ public class SystemPropertyConfig extends Config { + private final String prefix; /** - * @param prefix Prefix string prepended to config keys + * @param prefix prefix string prepended to config keys * as they are looked up as system properties. */ public SystemPropertyConfig(String prefix) { this.prefix = prefix; } - /** - * @return a config value for the specified key - */ + /** Returns a config value for the specified key */ public String get(String key, String defaultValue) { return System.getProperty(prefix + key, defaultValue); } @@ -31,4 +30,5 @@ public class SystemPropertyConfig extends Config { public String toString() { return "Prefix=" + prefix; } + } diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index fd1c0f10d27..c330bf04d56 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -26,7 +26,7 @@ MetricManager::ConsumerSpec::~ConsumerSpec() = default; time_t MetricManager::Timer::getTime() const { - return vespalib::count_s(vespalib::steady_clock::now().time_since_epoch()); + return vespalib::count_s(vespalib::system_clock::now().time_since_epoch()); } void diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index bdd3c31fa38..f0caa358cb8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -166,8 +166,8 @@ public class LoadBalancerProvisioner { reals.add(new Real(HostName.from(node.hostname()), ip)); } } - log.log(LogLevel.INFO, "Creating load balancer for " + cluster + " in " + application.toShortString() + - ", targeting: " + reals); + log.log(LogLevel.DEBUG, "Creating load balancer for " + cluster + " in " + application.toShortString() + + ", targeting: " + reals); try { return service.create(application, cluster, reals, force); } catch (Exception e) { diff --git a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java index be871031101..d10892cd44f 100644 --- a/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java +++ b/predicate-search-core/src/main/java/com/yahoo/search/predicate/PredicateQueryParser.java @@ -4,7 +4,6 @@ package com.yahoo.search.predicate; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import org.apache.commons.lang.ArrayUtils; import java.io.IOException; import java.util.Arrays; @@ -112,7 +111,7 @@ public class PredicateQueryParser { private static void skipToken(JsonParser parser, JsonToken... expected) throws IOException { JsonToken actual = parser.nextToken(); - if (!ArrayUtils.contains(expected, actual)) { + if (Arrays.stream(expected).noneMatch(e -> e.equals(actual))) { throw new IllegalArgumentException( String.format("Expected a token in %s, got %s (%s).", Arrays.toString(expected), actual, parser.getTokenLocation())); diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java index a23339aa532..982d2488105 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/utils/VespaFeedWriter.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.utils; +import com.google.common.html.HtmlEscapers; import com.yahoo.document.predicate.Predicate; -import org.apache.commons.lang.StringEscapeUtils; import java.io.BufferedWriter; import java.io.IOException; @@ -35,7 +35,7 @@ public class VespaFeedWriter extends BufferedWriter { try { this.append(String.format("<document documenttype=\"%2$s\" documentid=\"id:%1$s:%2$s::%3$d\">\n", namespace, documentType, id)); - this.append("<" + fieldName + ">" + StringEscapeUtils.escapeHtml(predicate.toString()) + "</" + fieldName + ">\n"); + this.append("<" + fieldName + ">" + HtmlEscapers.htmlEscaper().escape(predicate.toString()) + "</" + fieldName + ">\n"); this.append("</document>\n"); } catch (IOException e) { throw new RuntimeException(e); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java index 5d7e668a0de..24aa59955bc 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java @@ -9,9 +9,8 @@ import com.yahoo.document.predicate.PredicateHash; import com.yahoo.document.predicate.RangeEdgePartition; import com.yahoo.document.predicate.RangePartition; import com.yahoo.search.predicate.index.Feature; -import com.yahoo.search.predicate.index.conjunction.IndexableFeatureConjunction; import com.yahoo.search.predicate.index.IntervalWithBounds; -import org.apache.commons.lang.ArrayUtils; +import com.yahoo.search.predicate.index.conjunction.IndexableFeatureConjunction; import org.junit.Test; import java.util.Arrays; @@ -232,7 +231,7 @@ public class PredicateTreeAnnotatorTest { long hash = PredicateHash.hash64(feature); List<Integer> actualIntervals = r.intervalMap.get(hash); assertNotNull(actualIntervals); - assertArrayEquals(ArrayUtils.toPrimitive(expectedIntervals), Ints.toArray(actualIntervals)); + assertArrayEquals(Ints.toArray(Arrays.asList(expectedIntervals)), Ints.toArray(actualIntervals)); } private static void assertBoundsContains(PredicateTreeAnnotations r, String feature, IntervalWithBounds expectedBounds) { diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java index 31777959704..13a15cab52a 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.predicate.index; -import org.apache.commons.lang.ArrayUtils; +import com.google.common.primitives.Ints; import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.junit.Test; @@ -109,7 +109,7 @@ public class CachedPostingListCounterTest { private static PostingList postingList(Integer... docIds) { PostingList postingList = mock(PostingList.class); - when(postingList.getDocIds()).thenReturn(ArrayUtils.toPrimitive(docIds)); + when(postingList.getDocIds()).thenReturn(Ints.toArray(Arrays.asList((docIds)))); return postingList; } diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index 28d11f14489..b381ee9122f 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -6,7 +6,7 @@ #include <vespa/fastos/file.h> #include <vespa/searchcore/config/config-hwinfo.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib//util/time.h> +#include <vespa/vespalib/util/time.h> #include <filesystem> #include <thread> #include <vespa/log/log.h> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index 3a3410aeebb..2a6e6793bcd 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -11,10 +11,13 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; +import java.util.stream.Collectors; /** * A free, parametrized function @@ -27,7 +30,12 @@ public class LambdaFunctionNode extends CompositeNode { private final ExpressionNode functionExpression; public LambdaFunctionNode(List<String> arguments, ExpressionNode functionExpression) { - // TODO: Verify that the function only accesses the given arguments + if ( ! arguments.containsAll(featuresAccessedIn(functionExpression))) { + throw new IllegalArgumentException("Lambda " + functionExpression + " accesses features outside its scope: " + + featuresAccessedIn(functionExpression).stream() + .filter(f -> ! arguments.contains(f)) + .collect(Collectors.joining(", "))); + } this.arguments = ImmutableList.copyOf(arguments); this.functionExpression = functionExpression; } @@ -134,6 +142,22 @@ public class LambdaFunctionNode extends CompositeNode { }); } + private static Set<String> featuresAccessedIn(ExpressionNode node) { + if (node instanceof ReferenceNode) { + return Set.of(((ReferenceNode) node).reference().toString()); + } + else if (node instanceof NameNode) { // (This clause probably not necessary) + return Set.of(((NameNode) node).getValue()); + } + else if (node instanceof CompositeNode) { + Set<String> features = new HashSet<>(); + ((CompositeNode)node).children().forEach(child -> features.addAll(featuresAccessedIn(child))); + return features; + } + return Set.of(); + } + + private class DoubleUnaryLambda implements DoubleUnaryOperator { @Override diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 6e77ab186e8..6a87e0c6d46 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -459,6 +459,22 @@ public class EvaluationTestCase { } @Test + public void testLambdaValidation() { + EvaluationTester tester = new EvaluationTester(); + try { + tester.assertEvaluates("{ {d1:0}:1, {d1:1}:2, {d1:2 }:3 }", + "map(tensor0, f(x) (log10(x+sum(tensor0)))", "{ {d1:0}:10, {d1:1}:100, {d1:2}:1000 }"); + fail("Expected validation failure"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("Lambda log10(x + reduce(tensor0, sum)) accesses features outside its scope: tensor0", + e.getMessage()); + } + + } + + @Test public void testExpand() { EvaluationTester tester = new EvaluationTester(); // Add a dimension using a literal tensor diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java index 50f37486b90..2cfc8cb575e 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java @@ -26,6 +26,7 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[3]", "d0[3]"); assertWillOptimize("d0[1]", "d0[1]", "d0"); assertWillOptimize("d0[2]", "d0[2]", "d0"); + assertWillOptimize("d0[1]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3],d1[2]", "d0"); assertWillOptimize("d0[3],d1[2]", "d0[3]", "d0"); @@ -33,10 +34,10 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[2],d1[3]", "d1[3]", "d1"); assertWillOptimize("d0[2],d2[2]", "d1[3],d2[2]", "d2"); assertWillOptimize("d1[2],d2[2]", "d0[3],d2[2]", "d2"); - assertWillOptimize("d0[1],d2[4]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[2],d2[4]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]"); - assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]", "d0,d1"); + assertWillOptimize("d0[1],d2[2]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[2],d2[2]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]"); + assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]", "d0,d1"); assertWillOptimize("d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d1[2],d2[3]", "d2[3],d3[4],d4[5]", "d2"); diff --git a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java index 8bb7e0e5ab9..6e69c36d69c 100644 --- a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java @@ -100,6 +100,9 @@ public class KeyStoreBuilder { } catch (IOException e) { throw new UncheckedIOException(e); } + finally { + inputFilePassword = null; + } } private static class KeyEntry { @@ -125,4 +128,5 @@ public class KeyStoreBuilder { this.certificate = certificate; } } + } diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index 221018122bf..d2b98fd20d9 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -136,6 +136,9 @@ public class SslContextBuilder { } catch (IOException e) { throw new UncheckedIOException(e); } + finally { + keyStorePassword = null; + } } private static KeyStore createTrustStore(List<X509Certificate> caCertificates) { diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index e5810c4e7b2..7ad51a55bff 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -35,8 +35,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; /** - * VespaRecordWriter sends the output <key, value> to one or more Vespa - * endpoints. + * VespaRecordWriter sends the output <key, value> to one or more Vespa endpoints. * * @author lesters */ diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java index fce7afdb436..0e202d1f348 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java @@ -21,6 +21,7 @@ public class Runner { /** * Feed data from inputFile to session. + * * @param feedClient where to send data to * @param inputStream source of data * @param isJson if input stream is of json formatted data diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 53e708601d7..9c4b81da806 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -62,11 +62,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>3.4</version> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java index ae8413f226e..325c5492776 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java @@ -5,7 +5,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.json.JsonWriter; import com.yahoo.documentapi.DumpVisitorDataHandler; -import org.apache.commons.lang3.exception.ExceptionUtils; +import com.yahoo.exception.ExceptionUtils; import java.nio.charset.StandardCharsets; @@ -43,7 +43,7 @@ class LocalDataVisitorHandler extends DumpVisitorDataHandler { } } catch (Exception e) { synchronized (monitor) { - errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); } } } @@ -62,7 +62,7 @@ class LocalDataVisitorHandler extends DumpVisitorDataHandler { } } catch (Exception e) { synchronized (monitor) { - errors.append(ExceptionUtils.getStackTrace(e)).append("\n"); + errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 94a38a9d3db..0485689b15d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -21,13 +21,13 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.documentapi.metrics.DocumentOperationStatus; import com.yahoo.documentapi.metrics.DocumentOperationType; +import com.yahoo.exception.ExceptionUtils; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import com.yahoo.yolean.concurrent.ResourceFactory; -import org.apache.commons.lang3.exception.ExceptionUtils; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -146,7 +146,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (Exception e) { throw new RestApiException(Response.createErrorResponse( 500, - "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTrace(e), + "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.VISITOR_ERROR)); } @@ -175,7 +175,7 @@ public class OperationHandlerImpl implements OperationHandler { visitorControlHandler.waitUntilDone(); // VisitorParameters' session timeout implicitly triggers timeout failures. throwIfFatalVisitingError(visitorControlHandler, restUri); } catch (InterruptedException e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERRUPTED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERRUPTED)); } if (localDataVisitorHandler.getErrors().isEmpty()) { Optional<String> continuationToken; @@ -214,7 +214,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (DocumentAccessException documentException) { response = createErrorResponse(documentException, restUri); } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } @@ -236,7 +236,7 @@ public class OperationHandlerImpl implements OperationHandler { } catch (DocumentAccessException documentException) { response = createErrorResponse(documentException, restUri); } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); } finally { syncSessions.free(syncSession); } @@ -268,7 +268,7 @@ public class OperationHandlerImpl implements OperationHandler { response = Response.createErrorResponse(400, documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCEPTION); } } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); + response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); } finally { syncSessions.free(syncSession); } @@ -297,7 +297,7 @@ public class OperationHandlerImpl implements OperationHandler { return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name())); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); } finally { syncSessions.free(syncSession); } @@ -441,7 +441,7 @@ public class OperationHandlerImpl implements OperationHandler { try { params.setResumeToken(ProgressToken.fromSerializedString(options.continuation.get())); } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); + throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); } } return params; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java index 57f3ab20186..d3f05e18841 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java @@ -35,4 +35,5 @@ public class Main { private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() { return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable)); } + } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index 88a9a783f37..50f5c9d484c 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -27,9 +27,10 @@ import java.util.logging.Logger; /** * Example client using visiting * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>, based on work by <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + * @author Einar M R Rosenvinge */ public class VdsVisitTarget { + private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName()); private boolean printIds = false; @@ -275,4 +276,5 @@ public class VdsVisitTarget { } } } + } diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index d0c1abe061a..f631b3e1c58 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1278,7 +1278,6 @@ "public void <init>()", "public void <init>(com.yahoo.tensor.TensorType$Value)", "public varargs void <init>(com.yahoo.tensor.TensorType[])", - "public varargs void <init>(boolean, com.yahoo.tensor.TensorType[])", "public void <init>(java.lang.Iterable)", "public void <init>(com.yahoo.tensor.TensorType$Value, java.lang.Iterable)", "public int rank()", diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 32040785fc2..a8380e9513c 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -24,10 +24,6 @@ <artifactId>lz4</artifactId> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-exec</artifactId> </dependency> diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java index 479850beb1a..86ff9eacb8e 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java @@ -3,10 +3,14 @@ package com.yahoo.collections; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang.builder.ToStringBuilder; import java.lang.reflect.InvocationTargetException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * A map holding multiple items at each key (using ArrayList and HashMap). @@ -140,7 +144,10 @@ public class ListMap<K, V> { @Override public String toString() { - return ToStringBuilder.reflectionToString(this); + return "ListMap{" + + "frozen=" + frozen + + ", map=" + map + + '}'; } /** Returns the number of keys in this map */ diff --git a/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java new file mode 100644 index 00000000000..4d4a2d02a1a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java @@ -0,0 +1,40 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.exception; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.UncheckedIOException; + +/** + * Misc exception utility methods - replacement for Apache commons-lang's ExceptionUtils + * + * @author bjorncs + */ +public class ExceptionUtils { + + private ExceptionUtils() {} + + public static String getStackTraceAsString(Throwable throwable) { + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + throwable.printStackTrace(printWriter); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static String getStackTraceRecursivelyAsString(Throwable throwable) { + Throwable cause = throwable; + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true)) { + do { + cause.printStackTrace(printWriter); + } while ((cause = cause.getCause()) != null); + return stringWriter.getBuffer().toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/exception/package-info.java b/vespajlib/src/main/java/com/yahoo/exception/package-info.java new file mode 100644 index 00000000000..8254f90651f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/exception/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.exception; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/net/Url.java b/vespajlib/src/main/java/com/yahoo/net/Url.java index 60a9c98bce8..73584b4bc4e 100644 --- a/vespajlib/src/main/java/com/yahoo/net/Url.java +++ b/vespajlib/src/main/java/com/yahoo/net/Url.java @@ -250,4 +250,5 @@ public class Url { public String toString() { return image; } + } diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java index 77d931b5278..860107a4cc3 100644 --- a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java +++ b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java @@ -175,4 +175,5 @@ public class UrlTokenizer { schemeToPort.put(scheme, str); portToScheme.put(str, scheme); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index d8959147ee0..aeed8c33093 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -323,7 +323,7 @@ public class TensorType { * [N] + [] = [] * [] + {} = {} */ - Dimension combineWith(Optional<Dimension> other, boolean allowDifferentSizes) { + Dimension combineWith(Optional<Dimension> other) { if ( ! other.isPresent()) return this; if (this instanceof MappedDimension) return this; if (other.get() instanceof MappedDimension) return other.get(); @@ -333,11 +333,7 @@ public class TensorType { // both are indexed bound IndexedBoundDimension thisIb = (IndexedBoundDimension)this; IndexedBoundDimension otherIb = (IndexedBoundDimension)other.get(); - if (allowDifferentSizes) - return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; - if ( ! thisIb.size().equals(otherIb.size())) - throw new IllegalArgumentException("Unequal dimension sizes in " + thisIb + " and " + otherIb); - return thisIb; + return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; } @Override @@ -498,13 +494,9 @@ public class TensorType { * The value type will be the largest of the value types of the input types */ public Builder(TensorType ... types) { - this(true, types); - } - - public Builder(boolean allowDifferentSizes, TensorType ... types) { this.valueType = TensorType.combinedValueType(types); for (TensorType type : types) - addDimensionsOf(type, allowDifferentSizes); + addDimensionsOf(type); } /** Creates a builder from the given dimensions, having double as the value type */ @@ -522,17 +514,17 @@ public class TensorType { private static final boolean supportsMixedTypes = false; - private void addDimensionsOf(TensorType type, boolean allowDifferentSizes) { + private void addDimensionsOf(TensorType type) { if ( ! supportsMixedTypes) { // TODO: Support it - addDimensionsOfAndDisallowMixedDimensions(type, allowDifferentSizes); + addDimensionsOfAndDisallowMixedDimensions(type); } else { for (Dimension dimension : type.dimensions) - set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())), allowDifferentSizes)); + set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())))); } } - private void addDimensionsOfAndDisallowMixedDimensions(TensorType type, boolean allowDifferentSizes) { + private void addDimensionsOfAndDisallowMixedDimensions(TensorType type) { boolean containsMapped = dimensions.values().stream().anyMatch(d -> ! d.isIndexed()); containsMapped = containsMapped || type.dimensions().stream().anyMatch(d -> ! d.isIndexed()); @@ -540,7 +532,7 @@ public class TensorType { if (containsMapped) dimension = new MappedDimension(dimension.name()); Dimension existing = dimensions.get(dimension.name()); - set(dimension.combineWith(Optional.ofNullable(existing), allowDifferentSizes)); + set(dimension.combineWith(Optional.ofNullable(existing))); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 5419d04a4fb..1e0eaa7fad3 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -48,12 +48,7 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP /** Returns the type resulting from applying Join to the two given types */ public static TensorType outputType(TensorType a, TensorType b) { - try { - return new TensorType.Builder(false, a, b).build(); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Can not join " + a + " and " + b, e); - } + return new TensorType.Builder(a, b).build(); } public DoubleBinaryOperator combinator() { return combinator; } @@ -80,14 +75,14 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP @Override public TensorType type(TypeContext<NAMETYPE> context) { - return outputType(argumentA.type(context), argumentB.type(context)); + return new TensorType.Builder(argumentA.type(context), argumentB.type(context)).build(); } @Override public Tensor evaluate(EvaluationContext<NAMETYPE> context) { Tensor a = argumentA.evaluate(context); Tensor b = argumentB.evaluate(context); - TensorType joinedType = outputType(a.type(), b.type()); + TensorType joinedType = new TensorType.Builder(a.type(), b.type()).build(); return evaluate(a, b, joinedType, combinator); } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java index 73b40222f55..4d8b5012c81 100755 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java @@ -131,4 +131,5 @@ public class ObjectDumper extends ObjectVisitor { addLine(name + ": " + obj); } } + } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 5bd1bbdba37..b1851b5f120 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -130,11 +130,13 @@ public class TensorTestCase { assertEquals("Mapped vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.mapped, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(3, Type.indexedUnbound), vectors(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(5, Type.indexedUnbound), vectors(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(3, Type.indexedBound, 2))); + assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(5, Type.indexedBound, 2))); + assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(5, Type.indexedBound), vectors(3, Type.indexedBound, 2))); assertEquals("Mapped matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.mapped, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(3, Type.indexedUnbound), matrix(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(5, Type.indexedUnbound), matrix(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(3, Type.indexedBound, 2))); + assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(5, Type.indexedBound, 2))); + assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(5, Type.indexedBound), matrix(3, Type.indexedBound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.indexedUnbound, 2))); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 14c14fe85ca..9339cdacea0 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -78,6 +78,7 @@ vespa_define_module( src/tests/net/tls/transport_options src/tests/objects/nbostream src/tests/optimized + src/tests/overload src/tests/portal src/tests/portal/handle_manager src/tests/portal/http_request @@ -130,6 +131,7 @@ vespa_define_module( src/tests/util/md5 src/tests/util/rcuvector src/tests/valgrind + src/tests/visit_ranges src/tests/websocket src/tests/zcurve diff --git a/vespalib/src/tests/overload/CMakeLists.txt b/vespalib/src/tests/overload/CMakeLists.txt new file mode 100644 index 00000000000..67aa6230225 --- /dev/null +++ b/vespalib/src/tests/overload/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_overload_test_app TEST + SOURCES + overload_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_overload_test_app COMMAND vespalib_overload_test_app) diff --git a/vespalib/src/tests/overload/overload_test.cpp b/vespalib/src/tests/overload/overload_test.cpp new file mode 100644 index 00000000000..ceae29ac02f --- /dev/null +++ b/vespalib/src/tests/overload/overload_test.cpp @@ -0,0 +1,19 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <variant> +#include <string> + +using namespace vespalib; + +TEST(OverloadTest, visit_with_overload_works) { + std::variant<std::string,int> a = 10; + std::variant<std::string,int> b = "foo"; + std::visit(overload{[](int v){ EXPECT_EQ(v,10); }, + [](const std::string &){ FAIL() << "invalid visit"; }}, a); + std::visit(overload{[](int){ FAIL() << "invalid visit"; }, + [](const std::string &v){ EXPECT_EQ(v, "foo"); }}, b); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/visit_ranges/CMakeLists.txt b/vespalib/src/tests/visit_ranges/CMakeLists.txt new file mode 100644 index 00000000000..de94b2ebb1e --- /dev/null +++ b/vespalib/src/tests/visit_ranges/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_visit_ranges_test_app TEST + SOURCES + visit_ranges_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_visit_ranges_test_app COMMAND vespalib_visit_ranges_test_app) diff --git a/vespalib/src/tests/visit_ranges/visit_ranges_test.cpp b/vespalib/src/tests/visit_ranges/visit_ranges_test.cpp new file mode 100644 index 00000000000..cb1ac9bd9c3 --- /dev/null +++ b/vespalib/src/tests/visit_ranges/visit_ranges_test.cpp @@ -0,0 +1,120 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/overload.h> +#include <vespa/vespalib/util/visit_ranges.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <variant> +#include <string> + +using namespace vespalib; + +TEST(VisitRangeExample, set_intersection) { + std::vector<int> first({1,3,7}); + std::vector<int> second({2,3,8}); + std::vector<int> result; + vespalib::visit_ranges(overload{[](visit_ranges_either, int) {}, + [&result](visit_ranges_both, int x, int) { result.push_back(x); }}, + first.begin(), first.end(), second.begin(), second.end()); + EXPECT_EQ(result, std::vector<int>({3})); +} + +TEST(VisitRangeExample, set_subtraction) { + std::vector<int> first({1,3,7}); + std::vector<int> second({2,3,8}); + std::vector<int> result; + vespalib::visit_ranges(overload{[&result](visit_ranges_first, int a) { result.push_back(a); }, + [](visit_ranges_second, int) {}, + [](visit_ranges_both, int, int) {}}, + first.begin(), first.end(), second.begin(), second.end()); + EXPECT_EQ(result, std::vector<int>({1,7})); +} + +TEST(VisitRangesTest, empty_ranges_can_be_visited) { + std::vector<int> a; + std::vector<int> b; + std::vector<int> c; + auto visitor = overload + { + [&c](visit_ranges_either, int) { + c.push_back(42); + }, + [&c](visit_ranges_both, int, int) { + c.push_back(42); + } + }; + vespalib::visit_ranges(visitor, a.begin(), a.end(), b.begin(), b.end()); + EXPECT_EQ(c, std::vector<int>({})); +} + +TEST(VisitRangesTest, simple_merge_can_be_implemented) { + std::vector<int> a({1,3,7}); + std::vector<int> b({2,3,8}); + std::vector<int> c; + auto visitor = overload + { + [&c](visit_ranges_either, int x) { + c.push_back(x); + }, + [&c](visit_ranges_both, int x, int y) { + c.push_back(x); + c.push_back(y); + } + }; + vespalib::visit_ranges(visitor, a.begin(), a.end(), b.begin(), b.end()); + EXPECT_EQ(c, std::vector<int>({1,2,3,3,7,8})); +} + +TEST(VisitRangesTest, simple_union_can_be_implemented) { + std::vector<int> a({1,3,7}); + std::vector<int> b({2,3,8}); + std::vector<int> c; + auto visitor = overload + { + [&c](visit_ranges_either, int x) { + c.push_back(x); + }, + [&c](visit_ranges_both, int x, int) { + c.push_back(x); + } + }; + vespalib::visit_ranges(visitor, a.begin(), a.end(), b.begin(), b.end()); + EXPECT_EQ(c, std::vector<int>({1,2,3,7,8})); +} + +TEST(VisitRangesTest, asymmetric_merge_can_be_implemented) { + std::vector<int> a({1,3,7}); + std::vector<int> b({2,3,8}); + std::vector<int> c; + auto visitor = overload + { + [&c](visit_ranges_first, int x) { + c.push_back(x); + }, + [&c](visit_ranges_second, int) {}, + [&c](visit_ranges_both, int x, int y) { + c.push_back(x * y); + } + }; + vespalib::visit_ranges(visitor, a.begin(), a.end(), b.begin(), b.end()); + EXPECT_EQ(c, std::vector<int>({1,9,7})); +} + +TEST(VisitRangesTest, comparator_can_be_specified) { + std::vector<int> a({7,3,1}); + std::vector<int> b({8,3,2}); + std::vector<int> c; + auto visitor = overload + { + [&c](visit_ranges_either, int x) { + c.push_back(x); + }, + [&c](visit_ranges_both, int x, int y) { + c.push_back(x); + c.push_back(y); + } + }; + vespalib::visit_ranges(visitor, a.begin(), a.end(), b.begin(), b.end(), std::greater<>()); + EXPECT_EQ(c, std::vector<int>({8,7,3,3,2,1})); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/util/overload.h b/vespalib/src/vespa/vespalib/util/overload.h new file mode 100644 index 00000000000..d5af9dee2d3 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/overload.h @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib { + +/** + * Simple overload lambda composition class. To be replaced by + * standard overload functionality when available. (C++20) + **/ + +template<class... Ts> struct overload : Ts... { using Ts::operator()...; }; +template<class... Ts> overload(Ts...) -> overload<Ts...>; + +} diff --git a/vespalib/src/vespa/vespalib/util/visit_ranges.h b/vespalib/src/vespa/vespalib/util/visit_ranges.h new file mode 100644 index 00000000000..9cf1112fe0d --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/visit_ranges.h @@ -0,0 +1,81 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <functional> + +namespace vespalib { + +struct visit_ranges_either {}; +struct visit_ranges_first : visit_ranges_either {}; +struct visit_ranges_second : visit_ranges_either {}; +struct visit_ranges_both {}; + +/** + * Visit elements from two distinct ranges in the order defined by the + * given comparator. The comparator must define a strict-weak ordering + * across all elements from both ranges and each range must already be + * sorted according to the comparator before calling this + * function. Pairs of elements from the two ranges (one from each) + * that are equal according to the comparator will be visited by a + * single callback. The different cases ('from the first range', 'from + * the second range' and 'from both ranges') are indicated by using + * tagged dispatch in the visitation callback. + * + * An example treating both inputs equally: + * <pre> + * TEST(VisitRangeExample, set_intersection) { + * std::vector<int> first({1,3,7}); + * std::vector<int> second({2,3,8}); + * std::vector<int> result; + * vespalib::visit_ranges(overload{[](visit_ranges_either, int) {}, + * [&result](visit_ranges_both, int x, int) { result.push_back(x); }}, + * first.begin(), first.end(), second.begin(), second.end()); + * EXPECT_EQ(result, std::vector<int>({3})); + * } + * </pre> + * + * An example treating the inputs differently: + * <pre> + * TEST(VisitRangeExample, set_subtraction) { + * std::vector<int> first({1,3,7}); + * std::vector<int> second({2,3,8}); + * std::vector<int> result; + * vespalib::visit_ranges(overload{[&result](visit_ranges_first, int a) { result.push_back(a); }, + * [](visit_ranges_second, int) {}, + * [](visit_ranges_both, int, int) {}}, + * first.begin(), first.end(), second.begin(), second.end()); + * EXPECT_EQ(result, std::vector<int>({1,7})); + * } + * </pre> + * + * The intention of this function is to simplify the implementation of + * merge-like operations. + **/ + +template <typename V, typename ItA, typename ItB, typename Cmp = std::less<> > +void visit_ranges(V &&visitor, ItA pos_a, ItA end_a, ItB pos_b, ItB end_b, Cmp cmp = Cmp()) { + while ((pos_a != end_a) && (pos_b != end_b)) { + if (cmp(*pos_a, *pos_b)) { + visitor(visit_ranges_first(), *pos_a); + ++pos_a; + } else if (cmp(*pos_b, *pos_a)) { + visitor(visit_ranges_second(), *pos_b); + ++pos_b; + } else { + visitor(visit_ranges_both(), *pos_a, *pos_b); + ++pos_a; + ++pos_b; + } + } + while (pos_a != end_a) { + visitor(visit_ranges_first(), *pos_a); + ++pos_a; + } + while (pos_b != end_b) { + visitor(visit_ranges_second(), *pos_b); + ++pos_b; + } +} + +} |