From 8456f802f14447fa4d93154eb0945bc7867686a4 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 16 Dec 2019 09:18:19 +0100 Subject: Tensor parse tensor mapped 1-d short form {a:1, b:2, ...} --- .../main/java/com/yahoo/tensor/TensorParser.java | 96 +++++++++++++--------- .../com/yahoo/tensor/TensorParserTestCase.java | 8 ++ 2 files changed, 67 insertions(+), 37 deletions(-) (limited to 'vespajlib/src') diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index ea21249bede..5a1fd98a009 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -125,8 +125,7 @@ class TensorParser { valueString = valueString.trim(); if ( ! valueString.startsWith("{") && valueString.endsWith("}")) throw new IllegalArgumentException("A mixed tensor must be enclosed in {}"); - // TODO: Check if there is also at least one bound indexed dimension - MixedTensor.BoundBuilder builder = (MixedTensor.BoundBuilder)Tensor.Builder.of(type.get()); + Tensor.Builder builder = Tensor.Builder.of(type.get()); MixedValueParser parser = new MixedValueParser(valueString, dimensionOrder, builder); parser.parse(); return builder.build(); @@ -177,6 +176,40 @@ class TensorParser { position++; } + protected Number consumeNumber(TensorType.Value cellValueType) { + skipSpace(); + + int nextNumberEnd = nextStopCharIndex(position, string); + try { + String cellValueString = string.substring(position, nextNumberEnd); + try { + if (cellValueType == TensorType.Value.DOUBLE) + return Double.parseDouble(cellValueString); + else if (cellValueType == TensorType.Value.FLOAT) + return Float.parseFloat(cellValueString); + else + throw new IllegalArgumentException(cellValueType + " is not supported"); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("At value position " + position + ": '" + + cellValueString + "' is not a valid " + cellValueType); + } + } + finally { + position = nextNumberEnd; + } + } + + protected int nextStopCharIndex(int position, String valueString) { + while (position < valueString.length()) { + if (valueString.charAt(position) == ',') return position; + if (valueString.charAt(position) == ']') return position; + if (valueString.charAt(position) == '}') return position; + position++; + } + throw new IllegalArgumentException("Malformed tensor value '" + valueString + + "': Expected a ',', ']' or '}' after position " + position); + } + } /** A single-use dense tensor string parser */ @@ -186,8 +219,6 @@ class TensorParser { private final IndexedTensor.Indexes indexes; private final boolean hasInnerStructure; - private long tensorIndex = 0; - public DenseValueParser(String string, List dimensionOrder, IndexedTensor.DirectIndexBuilder builder) { @@ -227,44 +258,24 @@ class TensorParser { } protected void consumeNumber() { - skipSpace(); - - int nextNumberEnd = nextStopCharIndex(position, string); - TensorType.Value cellValueType = builder.type().valueType(); - String cellValueString = string.substring(position, nextNumberEnd); - try { - if (cellValueType == TensorType.Value.DOUBLE) - builder.cellByDirectIndex(indexes.toSourceValueIndex(), Double.parseDouble(cellValueString)); - else if (cellValueType == TensorType.Value.FLOAT) - builder.cellByDirectIndex(indexes.toSourceValueIndex(), Float.parseFloat(cellValueString)); - else - throw new IllegalArgumentException(cellValueType + " is not supported"); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("At value position " + position + ": '" + - cellValueString + "' is not a valid " + cellValueType); - } - position = nextNumberEnd; - } - - private int nextStopCharIndex(int position, String valueString) { - while (position < valueString.length()) { - if (valueString.charAt(position) == ',') return position; - if (valueString.charAt(position) == ']') return position; - position++; - } - throw new IllegalArgumentException("Malformed tensor value '" + valueString + - "': Expected a ',', ']' or '}' after position " + position); + Number number = consumeNumber(builder.type().valueType()); + if (builder.type().valueType() == TensorType.Value.DOUBLE) + builder.cellByDirectIndex(indexes.toSourceValueIndex(), (Double)number); + else if (builder.type().valueType() == TensorType.Value.FLOAT) + builder.cellByDirectIndex(indexes.toSourceValueIndex(), (Float)number); } } + /** + * Parses mixed tensor short forms {a:[1,2], ...} AND 1d mapped tensor short form {a:b, ...}. + */ private static class MixedValueParser extends ValueParser { - private final MixedTensor.BoundBuilder builder; + private final Tensor.Builder builder; private List dimensionOrder; - public MixedValueParser(String string, List dimensionOrder, MixedTensor.BoundBuilder builder) { + public MixedValueParser(String string, List dimensionOrder, Tensor.Builder builder) { super(string); this.dimensionOrder = dimensionOrder; this.builder = builder; @@ -282,13 +293,16 @@ class TensorParser { while (position + 1 < string.length()) { int labelEnd = string.indexOf(':', position); if (labelEnd <= position) - throw new IllegalArgumentException("A mixed tensor value must be on the form {sparse-label:[dense subspace], ...} "); + throw new IllegalArgumentException("A mixed tensor value must be on the form {sparse-label:[dense subspace], ...}, or {sparse-label:value, ...}"); String label = string.substring(position, labelEnd); position = labelEnd + 1; skipSpace(); TensorAddress mappedAddress = new TensorAddress.Builder(mappedSubtype).add(mappedDimension.name(), label).build(); - parseDenseSubspace(mappedAddress, dimensionOrder); + if (builder.type().rank() > 1) + parseDenseSubspace(mappedAddress, dimensionOrder); + else + consumeNumber(mappedAddress); if ( ! consumeOptional(',')) consume('}'); skipSpace(); @@ -298,7 +312,7 @@ class TensorParser { private void parseDenseSubspace(TensorAddress sparseAddress, List denseDimensionOrder) { DenseValueParser denseParser = new DenseValueParser(string.substring(position), denseDimensionOrder, - builder.denseSubspaceBuilder(sparseAddress)); + ((MixedTensor.BoundBuilder)builder).denseSubspaceBuilder(sparseAddress)); denseParser.parse(); position+= denseParser.position(); } @@ -315,6 +329,14 @@ class TensorParser { return true; } + private void consumeNumber(TensorAddress address) { + Number number = consumeNumber(builder.type().valueType()); + if (builder.type().valueType() == TensorType.Value.DOUBLE) + builder.cell(address, (Double)number); + else if (builder.type().valueType() == TensorType.Value.FLOAT) + builder.cell(address, (Float)number); + } + } private static class SparseValueParser extends ValueParser { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 583a964851f..6f9a5c13886 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -97,6 +97,14 @@ public class TensorParserTestCase { Tensor.from("tensor(key{}, x[2]):{a:[1, 2], b:[3, 4]}")); } + @Test + public void testSparseShortFormParsing() { + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(key{})")) + .cell(TensorAddress.ofLabels("a"), 1) + .cell(TensorAddress.ofLabels("b"), 2).build(), + Tensor.from("tensor(key{}):{a:1, b:2}")); + } + @Test public void testMixedWrongOrder() { assertEquals("Opposite order of dimensions", -- cgit v1.2.3