diff options
8 files changed, 135 insertions, 144 deletions
diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index 124783756ca..56ede8897ed 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -9,14 +9,24 @@ import com.yahoo.net.HostName; import com.yahoo.slime.JsonFormat; import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,8 +44,7 @@ public class ConfigUtils { private static final String doubleFormattedMin = new DecimalFormat("#.#", new DecimalFormatSymbols(Locale.ENGLISH)).format(-1e308); /** - * Computes Md5 hash of a list of strings. The only change to input lines before - * computing md5 is to skip empty lines. + * Computes Md5 hash of a list of strings. * * @param payload a config payload * @return the Md5 hash of the list, with lowercase letters @@ -47,30 +56,7 @@ public class ConfigUtils { } catch (IOException e) { throw new RuntimeException(e); } - MessageDigest md5 = getMd5Instance(); - md5.update(baos.toByteArray()); - return HexDump.toHexString(md5.digest()).toLowerCase(); - } - - /** - * Computes Md5 hash of a list of strings. The only change to input lines before - * computing md5 is to skip empty lines. - * - * @param lines A list of lines - * @return the Md5 hash of the list, with lowercase letters - */ - public static String getMd5(List<String> lines) { - StringBuilder sb = new StringBuilder(); - for (String line : lines) { - // Remove empty lines - line = line.trim(); - if (line.length() > 0) { - sb.append(line).append("\n"); - } - } - MessageDigest md5 = getMd5Instance(); - md5.update(Utf8.toBytes(sb.toString())); - return HexDump.toHexString(md5.digest()).toLowerCase(); + return getMd5(baos.toByteArray()); } /** @@ -80,14 +66,16 @@ public class ConfigUtils { * @return the Md5 hash of the input, with lowercase letters */ public static String getMd5(String input) { - MessageDigest md5 = getMd5Instance(); - md5.update(IOUtils.utf8ByteBuffer(input)); - return HexDump.toHexString(md5.digest()).toLowerCase(); + return getMd5(input.getBytes(StandardCharsets.UTF_8)); } public static String getMd5(Utf8Array input) { + return getMd5(input.getBytes()); + } + + public static String getMd5(byte[] input) { MessageDigest md5 = getMd5Instance(); - md5.update(input.getBytes()); + md5.update(input); return HexDump.toHexString(md5.digest()).toLowerCase(); } @@ -95,7 +83,7 @@ public class ConfigUtils { try { return MessageDigest.getInstance("MD5"); } catch (NoSuchAlgorithmException e) { - return null; + throw new RuntimeException("Could not get md5 instance"); } } @@ -161,12 +149,6 @@ public class ConfigUtils { } } - MessageDigest md5; - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - return null; - } StringBuilder sb = new StringBuilder(); for (String line : linesCopy) { // Normalize line, like it's done in make-config-preproc.pl @@ -193,8 +175,7 @@ public class ConfigUtils { sb.append(line).append("\n"); } } - md5.update(Utf8.toBytes(sb.toString())); - return HexDump.toHexString(md5.digest()).toLowerCase(); + return getMd5(sb.toString()); } /** diff --git a/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java b/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java index 5a5a9d1e100..190c7479ee7 100644 --- a/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java @@ -67,18 +67,18 @@ public class ConfigUtilsTest { lines.add("foo=\"1#hello\""); lines.add(""); //empty line should not affect md5sum - assertThat(ConfigUtils.getMd5(lines), is(expectedMd5)); + assertThat(getMd5(lines), is(expectedMd5)); lines.clear(); // Check that comment character in string leads to a different md5 than the original lines.add("foo=\"1#hello and some more\""); - String md5 = ConfigUtils.getMd5(lines); + String md5 = getMd5(lines); assertThat(md5, is(not(expectedMd5))); // Check that added characters aft comment character in string leads to a different md5 than above lines.add("foo=\"1#hello and some more and even more\""); - assertThat(ConfigUtils.getMd5(lines), is(not(md5))); + assertThat(getMd5(lines), is(not(md5))); } @Test @@ -223,4 +223,23 @@ public class ConfigUtilsTest { assertThat(def.getNamespace(), is("mynamespace")); } + /** + * Computes Md5 hash of a list of strings. The only change to input lines before + * computing md5 is to skip empty lines. + * + * @param lines A list of lines + * @return the Md5 hash of the list, with lowercase letters + */ + private static String getMd5(List<String> lines) { + StringBuilder sb = new StringBuilder(); + for (String line : lines) { + // Remove empty lines + line = line.trim(); + if (line.length() > 0) { + sb.append(line).append("\n"); + } + } + return ConfigUtils.getMd5(sb.toString()); + } + } diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java index ac6bbea617e..9a7800988dc 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfig.java @@ -18,13 +18,14 @@ public class MakeConfig { classBuilder = createClassBuilder(root, nd, properties); } - public static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, MakeConfigProperties properties) { + private static ClassBuilder createClassBuilder(InnerCNode root, NormalizedDefinition nd, MakeConfigProperties properties) { if (isCpp(properties)) return new CppClassBuilder(root, nd, properties.destDir, properties.dirInRoot); else return new JavaClassBuilder(root, nd, properties.destDir, properties.javaPackagePrefix); } + @SuppressWarnings("WeakerAccess") // Used by ConfigGenMojo public static boolean makeConfig(MakeConfigProperties properties) throws FileNotFoundException { for (File specFile : properties.specFiles) { String name = specFile.getName(); @@ -49,7 +50,7 @@ public class MakeConfig { /** * Generates the code and print it to this.out. */ - void buildClasses() { + private void buildClasses() { classBuilder.createConfigClasses(); } @@ -58,7 +59,7 @@ public class MakeConfig { out.println(" (default language for generated code is Java)"); } - public static void main(String[] args) throws IOException, InterruptedException { + public static void main(String[] args) throws IOException { try { MakeConfigProperties props = new MakeConfigProperties(); boolean success = makeConfig(props); @@ -81,7 +82,7 @@ public class MakeConfig { } private static boolean isCpp(MakeConfigProperties properties) { - return (properties.language.equals("cppng") || properties.language.equals("cpp")); + return properties.language.equals("cpp"); } // The Exceptions class below is copied from vespajlib/com.yahoo.protect.Exceptions @@ -100,7 +101,7 @@ public class MakeConfig { * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. */ - public static String toMessageString(Throwable t) { + static String toMessageString(Throwable t) { StringBuilder b = new StringBuilder(); String lastMessage = null; String message; diff --git a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java index 13807e63e53..da746bf3a01 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/MakeConfigProperties.java @@ -11,9 +11,10 @@ import java.util.StringTokenizer; * * @author gjoranv */ +@SuppressWarnings("WeakerAccess") // Used by ConfigGenMojo public class MakeConfigProperties { - private static final List<String> legalLanguages = Arrays.asList("java", "cpp", "cppng" ); + private static final List<String> legalLanguages = Arrays.asList("java", "cpp" ); final File destDir; final File[] specFiles; @@ -33,13 +34,14 @@ public class MakeConfigProperties { System.getProperty("config.packagePrefix")); } + @SuppressWarnings("WeakerAccess") // Used by ConfigGenMojo public MakeConfigProperties(String destDir, - String specFiles, - String language, - String dirInRoot, - String dumpTree, - String generateFrameworkCode, - String javaPackagePrefix) throws PropertyException { + String specFiles, + String language, + String dirInRoot, + String dumpTree, + String generateFrameworkCode, + String javaPackagePrefix) throws PropertyException { this.destDir = checkDestinationDir(destDir); this.specFiles = checkSpecificationFiles(specFiles); this.language = checkLanguage(language); diff --git a/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java b/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java index 501d7778fd7..7486d464e43 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/MakeConfigTest.java @@ -12,7 +12,7 @@ import org.junit.Test; public class MakeConfigTest { - File dest; + private File dest; @Before public void setUp() { @@ -29,8 +29,9 @@ public class MakeConfigTest { if (dir.isDirectory()) { String[] children = dir.list(); - for (int i = 0; i < children.length; i++) { - boolean success = recursiveDeleteDir(new File(dir, children[i])); + assert children != null; + for (String child : children) { + boolean success = recursiveDeleteDir(new File(dir, child)); if (!success) return false; } @@ -42,10 +43,8 @@ public class MakeConfigTest { @Test public void testProps() throws PropertyException { - long ts = System.currentTimeMillis(); System.setProperty("config.dumpTree", "true"); System.setProperty("config.useFramework", "true"); - System.setProperty("config.requireNamespace", "true"); System.setProperty("config.dest", dest.getAbsolutePath()); System.setProperty("config.spec", "src/test/resources/allfeatures.def"); MakeConfigProperties p = new MakeConfigProperties(); @@ -57,7 +56,6 @@ public class MakeConfigTest { System.setProperty("config.dumpTree", "false"); System.setProperty("config.useFramework", "false"); - System.setProperty("config.requireNamespace", "false"); System.setProperty("config.dest", dest.getAbsolutePath()); System.setProperty("config.spec", "src/test/resources/allfeatures.def,src/test/resources/bar.foo.def"); p = new MakeConfigProperties(); @@ -71,7 +69,6 @@ public class MakeConfigTest { public void testMake() throws IOException, InterruptedException { System.setProperty("config.dumpTree", "true"); System.setProperty("config.useFramework", "true"); - System.setProperty("config.requireNamespace", "true"); System.setProperty("config.dest", dest.getAbsolutePath()); System.setProperty("config.spec", "src/test/resources/allfeatures.def"); MakeConfig.main(new String[]{}); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index add72b28577..b845e8102dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -83,7 +83,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { if (controller().system() == SystemName.cd) { MetricsService.DeploymentMetrics configServerCollectedMetrics = new ConfigServerMetricsService(controller().configServer()) .getDeploymentMetrics(application.id(), deployment.zone()); - log.log(Level.INFO, String.format("Deployment metrics for application %s in zone %s. \nQPS: %d\nWPS: %d\n Doc count: %d\nQuery latency: %d\nWrite latency: %d\n", + log.log(Level.INFO, String.format("Deployment metrics for application %s in zone %s. \nQPS: %.2f\nWPS: %.2f\nDoc count: %d\nQuery latency: %.2f\nWrite latency: %.2f", application.id().serializedForm(), deployment.zone().value(), configServerCollectedMetrics.queriesPerSecond(), diff --git a/functions.cmake b/functions.cmake index eb29a9ffe22..a049276d845 100644 --- a/functions.cmake +++ b/functions.cmake @@ -132,7 +132,7 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) add_custom_command( OUTPUT ${CONFIG_H_PATH} ${CONFIG_CPP_PATH} - COMMAND java -Dconfig.spec=${CONFIG_DEF_PATH} -Dconfig.dest=${CONFIG_DEST_PARENT_DIR} -Dconfig.lang=cppng -Dconfig.requireNamespace=false -Dconfig.subdir=${CONFIG_DEST_DIRNAME} -Dconfig.dumpTree=false -Xms64m -Xmx64m -jar ${PROJECT_SOURCE_DIR}/configgen/target/configgen.jar + COMMAND java -Dconfig.spec=${CONFIG_DEF_PATH} -Dconfig.dest=${CONFIG_DEST_PARENT_DIR} -Dconfig.lang=cpp -Dconfig.subdir=${CONFIG_DEST_DIRNAME} -Dconfig.dumpTree=false -Xms64m -Xmx64m -jar ${PROJECT_SOURCE_DIR}/configgen/target/configgen.jar WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. MAIN_DEPENDENCY ${CONFIG_DEF_PATH} ) diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index 1727a59bc0d..b1d3356dd62 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -36,6 +36,7 @@ using search::fef::FeatureType; using namespace search::fef::indexproperties; +namespace search::features { namespace { template <typename X, typename Y> bool equals(X lhs, Y rhs) { @@ -53,17 +54,17 @@ isUndefined(T value, BasicType::Type type) { switch (type) { case BasicType::INT8: - return search::attribute::isUndefined<int8_t>(static_cast<int8_t>(value)); + return attribute::isUndefined<int8_t>(static_cast<int8_t>(value)); case BasicType::INT16: - return search::attribute::isUndefined<int16_t>(static_cast<int16_t>(value)); + return attribute::isUndefined<int16_t>(static_cast<int16_t>(value)); case BasicType::INT32: - return search::attribute::isUndefined<int32_t>(static_cast<int32_t>(value)); + return attribute::isUndefined<int32_t>(static_cast<int32_t>(value)); case BasicType::INT64: - return search::attribute::isUndefined<int64_t>(static_cast<int64_t>(value)); + return attribute::isUndefined<int64_t>(static_cast<int64_t>(value)); case BasicType::FLOAT: - return search::attribute::isUndefined<float>(static_cast<float>(value)); + return attribute::isUndefined<float>(static_cast<float>(value)); case BasicType::DOUBLE: - return search::attribute::isUndefined<double>(static_cast<double>(value)); + return attribute::isUndefined<double>(static_cast<double>(value)); default: return false; } @@ -77,28 +78,22 @@ isUndefined<vespalib::stringref>(vespalib::stringref, BasicType::Type) } template <typename T> -search::feature_t +feature_t considerUndefined(T value, BasicType::Type type) { if (isUndefined(value, type)) { - return search::attribute::getUndefined<search::feature_t>(); + return attribute::getUndefined<feature_t>(); } - return search::features::util::getAsFeature(value); + return util::getAsFeature(value); } template <> -search::feature_t +feature_t considerUndefined<ConstCharPtr>(ConstCharPtr value, BasicType::Type ) { - return search::features::util::getAsFeature(value); -} - + return util::getAsFeature(value); } - -namespace search::features { -namespace { - /** * Implements the executor for fetching values from a single or array attribute vector */ @@ -301,67 +296,6 @@ WeightedSetAttributeExecutor<BT, T>::execute(uint32_t docId) outputs().set_number(3, count); // count } -} - -AttributeBlueprint::AttributeBlueprint() : - fef::Blueprint("attribute"), - _attrName(), - _attrKey(), - _extra(), - _tensorType(ValueType::double_type()) -{ -} - -AttributeBlueprint::~AttributeBlueprint() = default; - -void -AttributeBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, - fef::IDumpFeatureVisitor &) const -{ -} - -bool -AttributeBlueprint::setup(const fef::IIndexEnvironment & env, - const fef::ParameterList & params) -{ - // params[0] = attribute name - // params[1] = index (array attribute) or key (weighted set attribute) - _attrName = params[0].getValue(); - _attrKey = createAttributeKey(_attrName); - if (params.size() == 2) { - _extra = params[1].getValue(); - } - vespalib::string attrType = type::Attribute::lookup(env.getProperties(), _attrName); - if (!attrType.empty()) { - _tensorType = ValueType::from_spec(attrType); - if (_tensorType.is_error()) { - LOG(error, "%s: invalid type: '%s'", getName().c_str(), attrType.c_str()); - } - } - FeatureType output_type = _tensorType.is_double() - ? FeatureType::number() - : FeatureType::object(_tensorType); - describeOutput("value", "The value of a single value attribute, " - "the value at the given index of an array attribute, " - "the given key of a weighted set attribute, or" - "the tensor of a tensor attribute", output_type); - if (!_tensorType.is_tensor()) { - describeOutput("weight", "The weight associated with the given key in a weighted set attribute."); - describeOutput("contains", "1 if the given key is present in a weighted set attribute, 0 otherwise."); - describeOutput("count", "Returns the number of elements in this array or weighted set attribute."); - } - env.hintAttributeAccess(_attrName); - return !_tensorType.is_error(); -} - -fef::Blueprint::UP -AttributeBlueprint::createInstance() const -{ - return std::make_unique<AttributeBlueprint>(); -} - -namespace { - template <typename T> struct SingleValueExecutorCreator { using AttrType = SingleValueNumericAttribute<T>; @@ -485,6 +419,63 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: } +AttributeBlueprint::AttributeBlueprint() : + fef::Blueprint("attribute"), + _attrName(), + _attrKey(), + _extra(), + _tensorType(ValueType::double_type()) +{ +} + +AttributeBlueprint::~AttributeBlueprint() = default; + +void +AttributeBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, + fef::IDumpFeatureVisitor &) const +{ +} + +bool +AttributeBlueprint::setup(const fef::IIndexEnvironment & env, + const fef::ParameterList & params) +{ + // params[0] = attribute name + // params[1] = index (array attribute) or key (weighted set attribute) + _attrName = params[0].getValue(); + _attrKey = createAttributeKey(_attrName); + if (params.size() == 2) { + _extra = params[1].getValue(); + } + vespalib::string attrType = type::Attribute::lookup(env.getProperties(), _attrName); + if (!attrType.empty()) { + _tensorType = ValueType::from_spec(attrType); + if (_tensorType.is_error()) { + LOG(error, "%s: invalid type: '%s'", getName().c_str(), attrType.c_str()); + } + } + FeatureType output_type = _tensorType.is_double() + ? FeatureType::number() + : FeatureType::object(_tensorType); + describeOutput("value", "The value of a single value attribute, " + "the value at the given index of an array attribute, " + "the given key of a weighted set attribute, or" + "the tensor of a tensor attribute", output_type); + if (!_tensorType.is_tensor()) { + describeOutput("weight", "The weight associated with the given key in a weighted set attribute."); + describeOutput("contains", "1 if the given key is present in a weighted set attribute, 0 otherwise."); + describeOutput("count", "Returns the number of elements in this array or weighted set attribute."); + } + env.hintAttributeAccess(_attrName); + return !_tensorType.is_error(); +} + +fef::Blueprint::UP +AttributeBlueprint::createInstance() const +{ + return std::make_unique<AttributeBlueprint>(); +} + void AttributeBlueprint::prepareSharedState(const fef::IQueryEnvironment & env, fef::IObjectStore & store) const { |