diff options
Diffstat (limited to 'vespajlib')
25 files changed, 633 insertions, 46 deletions
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index b8b6716d879..d9467a41f78 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1538,7 +1538,9 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.tensor.functions.TensorFunction)", "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.lang.String)", + "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.util.List)", "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", @@ -1553,7 +1555,9 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.tensor.functions.TensorFunction)", "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.lang.String)", + "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.util.List)", "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", @@ -2859,7 +2863,8 @@ ], "methods": [ "public static java.lang.String encode(java.util.Map)", - "public static java.lang.String escape(java.lang.String)" + "public static java.lang.String escape(java.lang.String)", + "public static boolean equals(java.lang.String, java.lang.String)" ], "fields": [] }, @@ -3022,7 +3027,8 @@ "public static boolean isTextCharacter(int)", "public static java.util.OptionalInt validateTextString(java.lang.String)", "public static boolean isDisplayable(int)", - "public static java.lang.String stripInvalidCharacters(java.lang.String)" + "public static java.lang.String stripInvalidCharacters(java.lang.String)", + "public static java.lang.String truncate(java.lang.String, int)" ], "fields": [] }, diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index a8380e9513c..7631a2af0fb 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -20,8 +20,8 @@ <!-- compile scope --> <dependency> - <groupId>net.jpountz.lz4</groupId> - <artifactId>lz4</artifactId> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> </dependency> <dependency> <groupId>org.apache.commons</groupId> diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index b7c6322d951..176e5044bc2 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -4,15 +4,14 @@ package com.yahoo.collections; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; -import static java.util.stream.Collectors.reducing; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -20,7 +19,7 @@ import static java.util.stream.Collectors.toUnmodifiableList; * * @author jonmv */ -public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> { +public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> implements Iterable<Type> { private final List<Type> items; private final boolean negate; @@ -63,7 +62,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte } /** Returns the union of the two lists. */ - public ListType and(ListType others) { + public ListType concat(ListType others) { return constructor.apply(Stream.concat(items.stream(), others.asList().stream()).collect(toUnmodifiableList()), false); } @@ -84,4 +83,9 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte public final int size() { return items.size(); } + @Override + public Iterator<Type> iterator() { + return items.iterator(); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/Pair.java b/vespajlib/src/main/java/com/yahoo/collections/Pair.java index 506ad10b98e..6587d1804f9 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/Pair.java +++ b/vespajlib/src/main/java/com/yahoo/collections/Pair.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.collections; +import java.util.Objects; + /** * An immutable pair of objects. This implements equals and hashCode by delegating to the * pair objects. @@ -33,19 +35,13 @@ public class Pair<F, S> { } @Override - public boolean equals(final Object o) { + public boolean equals(Object o) { if (o == this) return true; if (!(o instanceof Pair)) return false; @SuppressWarnings("rawtypes") - final Pair other = (Pair) o; - return equals(this.first, other.first) - && equals(this.second, other.second); - } - - private static boolean equals(final Object a, final Object b) { - if (a == null) return b == null; - return a.equals(b); + Pair other = (Pair) o; + return Objects.equals(this.first, other.first) && Objects.equals(this.second, other.second); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 9e9fac936f4..fb5da192f36 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -3,8 +3,12 @@ package com.yahoo.compress; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4SafeDecompressor; + import java.util.Arrays; import java.util.Optional; +import java.util.Random; /** * Compressor which can compress and decompress in various formats. @@ -19,7 +23,7 @@ public class Compressor { private final double compressionThresholdFactor; private final int compressMinSizeBytes; - private final LZ4Factory factory = LZ4Factory.fastestInstance(); + private static final LZ4Factory factory = LZ4Factory.fastestInstance(); /** Creates a compressor with default settings. */ public Compressor() { @@ -31,6 +35,10 @@ public class Compressor { this(type, 9, 0.95, 0); } + public Compressor(CompressionType type, int level) { + this(type, level, 0.95, 0); + } + /** * Creates a compressor. * @@ -79,8 +87,7 @@ public class Compressor { case LZ4: int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length; if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); - LZ4Compressor compressor = level < 7 ? factory.fastCompressor() : factory.highCompressor(); - byte[] compressedData = compressor.compress(data, 0, dataSize); + byte[] compressedData = getCompressor().compress(data, 0, dataSize); if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); return new Compression(CompressionType.LZ4, dataSize, compressedData); @@ -88,6 +95,9 @@ public class Compressor { throw new IllegalArgumentException(requestedCompression + " is not supported"); } } + private LZ4Compressor getCompressor() { + return level < 7 ? factory.fastCompressor() : factory.highCompressor(); + } /** Compresses some data using the requested compression type */ public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, Optional.empty()); } /** Compresses some data using the compression type of this compressor */ @@ -133,6 +143,39 @@ public class Compressor { return decompress(compression.type(), compression.data(), 0, compression.uncompressedSize(), Optional.empty()); } + public byte[] compressUnconditionally(byte[] input) { + return getCompressor().compress(input); + } + + public byte [] decompressUnconditionally(byte[] input, int srcOffset, int uncompressedLen) { + if (input.length > 0) { + return factory.fastDecompressor().decompress(input, srcOffset, uncompressedLen); + } + return new byte[0]; + } + + public long warmup(double seconds) { + byte [] input = new byte[0x4000]; + new Random().nextBytes(input); + long timeDone = System.nanoTime() + (long)(seconds*1000000000); + long compressedBytes = 0; + byte [] decompressed = new byte [input.length]; + LZ4FastDecompressor fastDecompressor = factory.fastDecompressor(); + LZ4SafeDecompressor safeDecompressor = factory.safeDecompressor(); + LZ4Compressor fastCompressor = factory.fastCompressor(); + LZ4Compressor highCompressor = factory.highCompressor(); + while (System.nanoTime() < timeDone) { + byte [] compressedFast = fastCompressor.compress(input); + byte [] compressedHigh = highCompressor.compress(input); + fastDecompressor.decompress(compressedFast, decompressed); + fastDecompressor.decompress(compressedHigh, decompressed); + safeDecompressor.decompress(compressedFast, decompressed); + safeDecompressor.decompress(compressedHigh, decompressed); + compressedBytes += compressedFast.length + compressedHigh.length; + } + return compressedBytes; + } + public static class Compression { private final CompressionType compressionType; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java new file mode 100644 index 00000000000..42e86aad1ba --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java @@ -0,0 +1,63 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * An executor that will first try a bounded cached threadpool before falling back to a unbounded + * single threaded threadpool that will take over dispatching to the primary pool. + * + */ +public class CachedThreadPoolWithFallback implements AutoCloseable, Executor { + private final ExecutorService primary; + private final ExecutorService secondary; + public CachedThreadPoolWithFallback(String baseName, int corePoolSize, int maximumPoolSize, long keepAlimeTime, TimeUnit timeUnit) { + primary = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAlimeTime, timeUnit, + new SynchronousQueue<>(), ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".primary")); + secondary = Executors.newSingleThreadExecutor(ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".secondary")); + } + @Override + public void execute(Runnable command) { + try { + primary.execute(command); + } catch (RejectedExecutionException e1) { + secondary.execute(() -> retryForever(command)); + } + } + private void retryForever(Runnable command) { + while (true) { + try { + primary.execute(command); + return; + } catch (RejectedExecutionException rejected) { + try { + Thread.sleep(1); + } catch (InterruptedException silenced) { } + } + } + } + + @Override + public void close() { + secondary.shutdown(); + join(secondary); + primary.shutdown(); + join(primary); + } + private static void join(ExecutorService executor) { + while (true) { + try { + if (executor.awaitTermination(60, TimeUnit.SECONDS)) { + return; + } + } catch (InterruptedException e) {} + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java index f199fefd185..f677ae23a45 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -3,10 +3,8 @@ package com.yahoo.slime; import com.yahoo.text.Text; import com.yahoo.text.Utf8; -import org.w3c.dom.CharacterData; import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; /** * A port of the C++ json decoder intended to be fast. @@ -47,6 +45,17 @@ public class JsonDecoder { return slime; } + /** Decode bytes as a UTF-8 JSON into Slime, or throw {@link JsonParseException} on invalid JSON. */ + public Slime decodeOrThrow(Slime slime, byte[] bytes) { + in = new BufferedInput(bytes); + next(); + decodeValue(slimeInserter.adjust(slime)); + if (in.failed()) { + throw new JsonParseException(in); + } + return slime; + } + private void decodeValue(Inserter inserter) { skipWhiteSpace(); switch (c) { diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java b/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java new file mode 100644 index 00000000000..6c42f7d38c1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * @author hakonhall + */ +public class JsonParseException extends RuntimeException { + + private static final long serialVersionUID = 1586949558L; + + private final BufferedInput input; + + JsonParseException(BufferedInput input) { + super(input.getErrorMessage()); + this.input = input; + } + + public byte[] getOffendingBytes() { + // potentially expensive array copy + return input.getOffending(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java index 8357e3035c0..83934e0c206 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -159,11 +159,10 @@ public final class Slime { } /** - * Tests whether this is equal to Inspector. + * Tests whether the two Inspectors are equal. * - * Since equality of two Inspectors is subtle, {@link Object#equals(Object)} is not used. + * <p>Since equality of two Inspectors is subtle, {@link Object#equals(Object)} is not used.</p> * - * @param that inspector. * @return true if they are equal. */ public boolean equalTo(Slime that) { diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java new file mode 100644 index 00000000000..2ed7331a60c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -0,0 +1,121 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * Extra utilities/operations on slime trees. + * + * @author Ulf Lilleengen + */ +public class SlimeUtils { + + public static void copyObject(Inspector from, Cursor to) { + if (from.type() != Type.OBJECT) { + throw new IllegalArgumentException("Cannot copy object: " + from); + } + from.traverse((ObjectTraverser) (name, inspector) -> setObjectEntry(inspector, name, to)); + + } + + private static void setObjectEntry(Inspector from, String name, Cursor to) { + switch (from.type()) { + case NIX: + to.setNix(name); + break; + case BOOL: + to.setBool(name, from.asBool()); + break; + case LONG: + to.setLong(name, from.asLong()); + break; + case DOUBLE: + to.setDouble(name, from.asDouble()); + break; + case STRING: + to.setString(name, from.asString()); + break; + case DATA: + to.setData(name, from.asData()); + break; + case ARRAY: + Cursor array = to.setArray(name); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.setObject(name); + copyObject(from, object); + break; + } + } + + private static void copyArray(Inspector from, final Cursor to) { + from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to)); + } + + private static void addValue(Inspector from, Cursor to) { + switch (from.type()) { + case NIX: + to.addNix(); + break; + case BOOL: + to.addBool(from.asBool()); + break; + case LONG: + to.addLong(from.asLong()); + break; + case DOUBLE: + to.addDouble(from.asDouble()); + break; + case STRING: + to.addString(from.asString()); + break; + case DATA: + to.addData(from.asData()); + break; + case ARRAY: + Cursor array = to.addArray(); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.addObject(); + copyObject(from, object); + break; + } + + } + + public static byte[] toJsonBytes(Slime slime) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new JsonFormat(true).encode(baos, slime); + return baos.toByteArray(); + } + + public static Slime jsonToSlime(byte[] json) { + Slime slime = new Slime(); + new JsonDecoder().decode(slime, json); + return slime; + } + + public static Slime jsonToSlime(String json) { + return jsonToSlime(json.getBytes(StandardCharsets.UTF_8)); + } + + /** Throws {@link JsonParseException} on invalid JSON. */ + public static Slime jsonToSlimeOrThrow(String json) { + return jsonToSlimeOrThrow(json.getBytes(StandardCharsets.UTF_8)); + } + + public static Slime jsonToSlimeOrThrow(byte[] json) { + Slime slime = new Slime(); + new JsonDecoder().decodeOrThrow(slime, json); + return slime; + } + + public static Optional<String> optionalString(Inspector inspector) { + return Optional.of(inspector.asString()).filter(s -> !s.isEmpty()); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java index cceac7e84bb..c455929bf51 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -16,6 +16,14 @@ import com.yahoo.collections.Pair; */ public class ProcessExecuter { + private final boolean override_log_control; + public ProcessExecuter(boolean override_log_control) { + this.override_log_control = override_log_control; + } + public ProcessExecuter() { + this(false); + } + /** * Executes the given command synchronously without timeout. * @@ -39,6 +47,10 @@ public class ProcessExecuter { ProcessBuilder pb = new ProcessBuilder(command); StringBuilder ret = new StringBuilder(); pb.environment().remove("VESPA_LOG_TARGET"); + if (override_log_control) { + pb.environment().remove("VESPA_LOG_CONTROL_FILE"); + pb.environment().put("VESPA_SERVICE_NAME", "exec-" + command[0]); + } pb.redirectErrorStream(true); Process p = pb.start(); InputStream is = p.getInputStream(); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java index a365f0f4bdc..a4b68a662da 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; +import com.google.common.collect.ImmutableList; import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -12,11 +14,20 @@ import java.util.List; public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYPE> { private final TensorFunction<NAMETYPE> argument; - private final String dimension; + private final List<String> dimensions; + + public Argmax(TensorFunction<NAMETYPE> argument) { + this(argument, Collections.emptyList()); + } public Argmax(TensorFunction<NAMETYPE> argument, String dimension) { + this(argument, Collections.singletonList(dimension)); + } + + public Argmax(TensorFunction<NAMETYPE> argument, List<String> dimensions) { + Objects.requireNonNull(dimensions, "The dimensions cannot be null"); this.argument = argument; - this.dimension = dimension; + this.dimensions = ImmutableList.copyOf(dimensions); } @Override @@ -24,22 +35,21 @@ public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { - if ( arguments.size() != 1) + if (arguments.size() != 1) throw new IllegalArgumentException("Argmax must have 1 argument, got " + arguments.size()); - return new Argmax<>(arguments.get(0), dimension); + return new Argmax<>(arguments.get(0), dimensions); } @Override public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { TensorFunction<NAMETYPE> primitiveArgument = argument.toPrimitive(); - return new Join<>(primitiveArgument, - new Reduce<>(primitiveArgument, Reduce.Aggregator.max, dimension), - ScalarFunctions.equal()); + TensorFunction<NAMETYPE> reduce = new Reduce<>(primitiveArgument, Reduce.Aggregator.max, dimensions); + return new Join<>(primitiveArgument, reduce, ScalarFunctions.equal()); } @Override public String toString(ToStringContext context) { - return "argmax(" + argument.toString(context) + ", " + dimension + ")"; + return "argmax(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java index 32ccdf51336..ad14bc1f1f2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; +import com.google.common.collect.ImmutableList; import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -12,11 +14,20 @@ import java.util.List; public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYPE> { private final TensorFunction<NAMETYPE> argument; - private final String dimension; + private final List<String> dimensions; + + public Argmin(TensorFunction<NAMETYPE> argument) { + this(argument, Collections.emptyList()); + } public Argmin(TensorFunction<NAMETYPE> argument, String dimension) { + this(argument, Collections.singletonList(dimension)); + } + + public Argmin(TensorFunction<NAMETYPE> argument, List<String> dimensions) { + Objects.requireNonNull(dimensions, "The dimensions cannot be null"); this.argument = argument; - this.dimension = dimension; + this.dimensions = ImmutableList.copyOf(dimensions); } @Override @@ -24,22 +35,21 @@ public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { - if ( arguments.size() != 1) + if (arguments.size() != 1) throw new IllegalArgumentException("Argmin must have 1 argument, got " + arguments.size()); - return new Argmin<>(arguments.get(0), dimension); + return new Argmin<>(arguments.get(0), dimensions); } @Override public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { TensorFunction<NAMETYPE> primitiveArgument = argument.toPrimitive(); - return new Join<>(primitiveArgument, - new Reduce<>(primitiveArgument, Reduce.Aggregator.min, dimension), - ScalarFunctions.equal()); + TensorFunction<NAMETYPE> reduce = new Reduce<>(primitiveArgument, Reduce.Aggregator.min, dimensions); + return new Join<>(primitiveArgument, reduce, ScalarFunctions.equal()); } @Override public String toString(ToStringContext context) { - return "argmin(" + argument.toString(context) + ", " + dimension + ")"; + return "argmin(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java index 4d3989b8782..bccd66acd31 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java @@ -230,7 +230,7 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY @Override public String toString() { - return toString(null); + return toString(ToStringContext.empty()); } public String toString(ToStringContext context) { diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java index cfff16c9aba..2757bd7945c 100644 --- a/vespajlib/src/main/java/com/yahoo/text/JSON.java +++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java @@ -1,10 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + import java.util.Map; /** - * Static methods for working with the map textual format which is parsed by {@link MapParser} + * Static methods for working with JSON. * * @author bratseth */ @@ -56,4 +59,20 @@ public final class JSON { return b != null ? b.toString() : s; } + /** + * Test whether two JSON strings are equal, e.g. the order of fields in an object is irrelevant. + * + * <p>When comparing two numbers of the two JSON strings, the result is only guaranteed to be + * correct if (a) both are integers (without fraction and exponent) and each fits in a long, or + * (b) both are non-integers, fits in a double, and are syntactically identical. Examples + * of pairs that may not be equal: 1 and 1.0 (different types), 0.1 and 1e-1, 0.0 and 0.00.</p> + * + * @throws RuntimeException on invalid JSON + */ + public static boolean equals(String left, String right) { + Slime leftSlime = SlimeUtils.jsonToSlimeOrThrow(left); + Slime rightSlime = SlimeUtils.jsonToSlimeOrThrow(right); + return leftSlime.equalTo(rightSlime); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 706fd1583a3..85b28639d89 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -174,4 +174,16 @@ public final class Text { return stripped != null ? stripped.toString() : string; } + /** + * Returns a string which is never larger than the given number of characters. + * If the string is longer than the given length it will be truncated. + * If length is 4 or less the string will be truncated to length. + * If length is longer than 4, it will be truncated at length-4 with " ..." added at the end. + */ + public static String truncate(String s, int length) { + if (s.length() <= length) return s; + if (length <= 4) return s.substring(0, length); + return s.substring(0, length - 4) + " ..."; + } + } diff --git a/vespajlib/src/main/java/net/jpountz/lz4/package-info.java b/vespajlib/src/main/java/net/jpountz/lz4/package-info.java new file mode 100644 index 00000000000..3536beff420 --- /dev/null +++ b/vespajlib/src/main/java/net/jpountz/lz4/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) +package net.jpountz.lz4; +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/main/java/net/jpountz/util/package-info.java b/vespajlib/src/main/java/net/jpountz/util/package-info.java new file mode 100644 index 00000000000..f09b2dde726 --- /dev/null +++ b/vespajlib/src/main/java/net/jpountz/util/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) +package net.jpountz.util; +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java b/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java new file mode 100644 index 00000000000..e8ad7b05556 --- /dev/null +++ b/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) +package net.jpountz.xxhash; +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java index 9386bf7256f..3524f507701 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java +++ b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java @@ -48,8 +48,8 @@ public class AbstractFilteringListTest { assertEquals(List.of("abc", "cba", "bbb"), list.not().in(MyList.of("ABC", "CBA")).asList()); - assertEquals(List.of("ABC", "abc", "cba", "bbb", "ABC", "aaa"), - list.and(MyList.of("aaa")).asList()); + assertEquals(List.of("ABC", "abc", "cba", "bbb", "ABC", "aaa", "ABC"), + list.concat(MyList.of("aaa", "ABC")).asList()); } private static class MyList extends AbstractFilteringList<String, MyList> { diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java new file mode 100644 index 00000000000..52e17631a34 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java @@ -0,0 +1,43 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.concurrent; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; + +public class CachedThreadPoolWithFallbackTest { + private static void countAndBlock(AtomicLong counter, long waitLimit) { + counter.incrementAndGet(); + try { + synchronized (counter) { + while (counter.get() < waitLimit) { + counter.wait(); + } + } + } catch (InterruptedException e) {} + } + + @Test + public void testThatTaskAreQueued() throws InterruptedException { + CachedThreadPoolWithFallback executor = new CachedThreadPoolWithFallback("test", 1, 30, 1, TimeUnit.SECONDS); + AtomicLong counter = new AtomicLong(0); + for (int i = 0; i < 1000; i++) { + executor.execute(() -> countAndBlock(counter, 100)); + } + while (counter.get() < 30) { + Thread.sleep(1); + } + Thread.sleep(1); + assertEquals(30L, counter.get()); + counter.set(100); + synchronized (counter) { + counter.notifyAll(); + } + executor.close(); + assertEquals(1070L, counter.get()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java new file mode 100644 index 00000000000..237b1575bfb --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java @@ -0,0 +1,100 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Ulf Lilleengen + */ +public class SlimeUtilsTest { + + @Test + public void test_copying_slime_types_into_cursor() { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + Cursor subobj = root.setObject("bar"); + + Slime slime2 = new Slime(); + Cursor root2 = slime2.setObject(); + root2.setString("a", "a"); + root2.setLong("b", 2); + root2.setBool("c", true); + root2.setDouble("d", 3.14); + root2.setData("e", new byte[]{0x64}); + root2.setNix("f"); + + SlimeUtils.copyObject(slime2.get(), subobj); + + assertThat(root.toString(), is("{\"foo\":\"foobie\",\"bar\":{\"a\":\"a\",\"b\":2,\"c\":true,\"d\":3.14,\"e\":\"0x64\",\"f\":null}}")); + } + + @Test + public void test_copying_slime_arrays_into_cursor() { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + Cursor subobj = root.setObject("bar"); + + Slime slime2 = new Slime(); + Cursor root2 = slime2.setObject(); + Cursor array = root2.setArray("a"); + array.addString("foo"); + array.addLong(4); + array.addBool(true); + array.addDouble(3.14); + array.addNix(); + array.addData(new byte[]{0x64}); + Cursor objinner = array.addObject(); + objinner.setString("inner", "binner"); + + SlimeUtils.copyObject(slime2.get(), subobj); + + assertThat(root.toString(), is("{\"foo\":\"foobie\",\"bar\":{\"a\":[\"foo\",4,true,3.14,null,\"0x64\",{\"inner\":\"binner\"}]}}")); + } + + @Test + public void test_slime_to_json() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + root.setObject("bar"); + String json = Utf8.toString(SlimeUtils.toJsonBytes(slime)); + assertThat(json, is("{\"foo\":\"foobie\",\"bar\":{}}")); + } + + @Test + public void test_json_to_slime() { + byte[] json = Utf8.toBytes("{\"foo\":\"foobie\",\"bar\":{}}"); + Slime slime = SlimeUtils.jsonToSlime(json); + assertThat(slime.get().field("foo").asString(), is("foobie")); + assertTrue(slime.get().field("bar").valid()); + } + + @Test + public void test_json_to_slime_or_throw() { + Slime slime = SlimeUtils.jsonToSlimeOrThrow("{\"foo\":\"foobie\",\"bar\":{}}"); + assertThat(slime.get().field("foo").asString(), is("foobie")); + assertTrue(slime.get().field("bar").valid()); + } + + @Test + public void test_invalid_json() { + try { + SlimeUtils.jsonToSlimeOrThrow("foo"); + fail(); + } catch (RuntimeException e) { + assertEquals("Unexpected character 'o'", e.getMessage()); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java index 625d5d44b19..05f7d27907c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java @@ -21,6 +21,8 @@ public class TensorFunctionTestCase { new Diag<>(new TensorType.Builder().indexed("y",3).indexed("x",2).indexed("z",4).build())); assertTranslated("join(tensor(x{}):{1:1.0,3:5.0,9:3.0}, reduce(tensor(x{}):{1:1.0,3:5.0,9:3.0}, max, x), f(a,b)(a==b))", new Argmax<>(new ConstantTensor<>("{ {x:1}:1, {x:3}:5, {x:9}:3 }"), "x")); + assertTranslated("join(tensor(x{}):{1:1.0,3:5.0,9:3.0}, reduce(tensor(x{}):{1:1.0,3:5.0,9:3.0}, max), f(a,b)(a==b))", + new Argmax<>(new ConstantTensor<>("{ {x:1}:1, {x:3}:5, {x:9}:3 }"))); } private void assertTranslated(String expectedTranslation, TensorFunction<Name> inputFunction) { diff --git a/vespajlib/src/test/java/com/yahoo/text/JSONTest.java b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java index 22174761571..fbd0f9d0403 100644 --- a/vespajlib/src/test/java/com/yahoo/text/JSONTest.java +++ b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java @@ -2,11 +2,16 @@ package com.yahoo.text; import org.junit.Test; -import static org.junit.Assert.assertEquals; import java.util.LinkedHashMap; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * @author bratseth */ @@ -22,4 +27,88 @@ public class JSONTest { assertEquals("{\"a \\\"key\\\"\":3,\"key2\":\"value\",\"key3\":3.3}", JSON.encode(map)); } + @Test + public void testEquals() { + assertTrue(JSON.equals("{}", "{}")); + + // Whitespace is irrelevant + assertTrue(JSON.equals("{}", "\n{ }")); + + // Order of fields in object is irrelevant + assertTrue(JSON.equals("{\"a\":0, \"c\":1}", "{\"c\":1, \"a\":0}")); + + // Object equality is not using subset + assertFalse(JSON.equals("{\"a\":0}", "{\"a\":0, \"b\":0}")); + assertFalse(JSON.equals("{\"a\":0, \"b\":0}", "{\"a\":0}")); + + // Order of elements of array is significant + assertFalse(JSON.equals("[\"a\",\"b\"]", "[\"b\",\"a\"]")); + + // Verify null-valued fields are not ignored + assertFalse(JSON.equals("{\"a\":null}", "{}")); + + // Current impl uses BigInteger if integer doesn't fit in a long. + assertEquals(9223372036854775807L, Long.MAX_VALUE); + assertTrue(JSON.equals("{\"a\": 9223372036854775807}", "{\"a\": 9223372036854775807}")); + + // double 1.0 and int 1 are different + assertTrue(JSON.equals("{\"a\": 1}", "{\"a\": 1}")); + assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\": 1.0}")); + assertFalse(JSON.equals("{\"a\": 1.0}", "{\"a\": 1}")); + + // Double-precision on numbers. Constant from Math.E. + assertTrue(JSON.equals("{\"e\": 2.71828182845904}", "{\"e\": 2.71828182845904}")); + + // Double.MAX_VALUE is 1.7976931348623157e+308 + assertTrue(JSON.equals("{\"e\": 1.7976931348623156e+308}", "{\"e\": 1.7976931348623156e+308}")); + + // Justification of above float values + double e1 = 2.7182818284590452354; + double e2 = 2.718281828459045; + double e3 = 2.71828182845904; + assertEquals(e1, Math.E, -1); + assertEquals(e1, e2, -1); + assertNotEquals(e1, e3, -1); + + // Invalid JSON throws RuntimeException + assertRuntimeException(() -> JSON.equals("", "{}")); + assertRuntimeException(() -> JSON.equals("{}", "")); + assertRuntimeException(() -> JSON.equals("{", "{}")); + assertRuntimeException(() -> JSON.equals("{}", "{")); + } + + @Test + public void implementationSpecificEqualsBehavior() { + // Exception thrown if outside a long + assertTrue( JSON.equals("{\"a\": 9223372036854775807}", "{\"a\": 9223372036854775807}")); + assertRuntimeException(() -> JSON.equals("{\"a\": 9223372036854775808}", "{\"a\": 9223372036854775808}")); + + // Infinity if floating point number outside of double, and hence equal + assertTrue(JSON.equals("{\"a\": 2.7976931348623158e+308}", "{\"a\": 2.7976931348623158e+308}")); + + // Ignores extraneous precision + assertTrue(JSON.equals( "{\"e\": 2.7182818284590452354}", + "{\"e\": 2.7182818284590452354}")); + assertTrue(JSON.equals( "{\"e\": 2.7182818284590452354}", + "{\"e\": 2.7182818284590452355}")); + assertFalse(JSON.equals("{\"e\": 2.7182818284590452354}", + "{\"e\": 2.71828182845904}")); + + // Comparing equal but syntactically different numbers + assertFalse(JSON.equals("{\"a\": 1.0}", "{\"a\":1}")); + assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\":1.00}")); + assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\":1.0000000000000000000000000000}")); + assertTrue(JSON.equals("{\"a\": 10.0}", "{\"a\":1e1}")); + assertTrue(JSON.equals("{\"a\": 1.2}", "{\"a\":12e-1}")); + } + + private static void assertRuntimeException(Runnable runnable) { + try { + runnable.run(); + fail("Expected RuntimeException to be thrown, but no exception was thrown"); + } catch (RuntimeException e) { + // OK + } + } + } diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index e733b838c39..8bb8b2aaad5 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -61,4 +61,15 @@ public class TextTestCase { assertFalse(Text.isDisplayable(0)); } + @Test + public void testTruncate() { + assertEquals("ab", Text.truncate("ab", 5)); + assertEquals("ab", Text.truncate("ab", 6)); + assertEquals("ab", Text.truncate("ab", 2)); + assertEquals("a", Text.truncate("ab", 1)); + assertEquals("", Text.truncate("ab", 0)); + assertEquals("ab c", Text.truncate("ab cde", 4)); + assertEquals("a ...", Text.truncate("ab cde", 5)); + } + } |