diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-10-05 15:04:38 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-10-05 15:04:38 +0200 |
commit | 3f96fe4e1a96b3b21c913a392da87e5ec669b286 (patch) | |
tree | bfcfbad0db1a54372fec7e483749706bdc55f6f6 /vespajlib/src | |
parent | 233b521555e240458e38f555430459c3e30f0d64 (diff) |
Revert "Remove unused Utf8 methods"
Diffstat (limited to 'vespajlib/src')
10 files changed, 83 insertions, 56 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index 266b7857363..f2de0ace476 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -1,24 +1,26 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; import java.io.*; import java.nio.channels.FileChannel; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.util.List; import java.nio.charset.Charset; import java.nio.ByteBuffer; + /** - * Some static I/O convenience methods. + * <p>Some static io convenience methods.</p> * - * @author bratseth - * @author Bjorn Borud + * @author bratseth + * @author Bjorn Borud */ public abstract class IOUtils { + static private final Charset utf8Charset = Charset.forName("utf-8"); + /** Closes a writer, or does nothing if the writer is null */ public static void closeWriter(Writer writer) { if (writer == null) return; @@ -327,7 +329,7 @@ public abstract class IOUtils { * Encodes string as UTF-8 into ByteBuffer */ public static ByteBuffer utf8ByteBuffer(String s) { - return StandardCharsets.UTF_8.encode(s); + return utf8Charset.encode(s); } /** @@ -339,7 +341,7 @@ public abstract class IOUtils { public static String readFile(File file) throws IOException { try { if (file == null) return null; - return Files.readString(file.toPath()); + return new String(Files.readAllBytes(file.toPath()), "utf-8"); } catch (NoSuchFileException e) { throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'"); diff --git a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java index f8a820a7421..6e7f9f5e50e 100644 --- a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java +++ b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java @@ -1,9 +1,10 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; import java.nio.Buffer; +import java.nio.charset.Charset; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; + /** * Conventient utility for reading lines from ByteBuffers. Please @@ -11,10 +12,11 @@ import java.nio.charset.StandardCharsets; * ByteBuffer abstraction is somewhat clumsy and thus usage of this * code requires that you understand the semantics clearly. * - * @author Bjorn Borud + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> * */ public class ReadLine { + static private Charset charset = Charset.forName("latin1"); /** * Extract next line from a byte buffer. Looks for EOL characters @@ -47,7 +49,7 @@ public class ReadLine { // The downcast to Buffer is done to avoid "redundant cast" warning on Java 9. // TODO: when Java 8 is gone, remove the casts and above comments. // extract string between start and i. - String line = StandardCharsets.ISO_8859_1.decode((ByteBuffer) ((Buffer)buffer.slice()).limit(i - start)).toString(); + String line = charset.decode((ByteBuffer) ((Buffer)buffer.slice()).limit(i - start)).toString(); // skip remaining for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) { @@ -78,5 +80,4 @@ public class ReadLine { static boolean isEolChar(byte b) { return ((10 == b) || (13 == b)); } - } diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java index 74458be4584..370d079b3ec 100644 --- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -1,9 +1,9 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; import com.google.common.collect.ImmutableSet; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import java.util.Collections; import java.util.List; import java.io.ByteArrayOutputStream; @@ -19,6 +19,8 @@ import java.util.Set; // TODO: Text utilities should which are still needed should move to Text. This should be deprecated. public class StringUtilities { + private static Charset UTF8 = Charset.forName("utf8"); + private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); } private static class ReplacementCharacters { @@ -63,7 +65,7 @@ public class StringUtilities { * @return The escaped string */ public static String escape(String source, char delimiter) { - byte bytes[] = source.getBytes(StandardCharsets.UTF_8); + byte bytes[] = source.getBytes(UTF8); ByteArrayOutputStream result = new ByteArrayOutputStream(); for (byte b : bytes) { int val = b; @@ -84,11 +86,11 @@ public class StringUtilities { result.write(replacementCharacters.replacement2[val]); } } - return new String(result.toByteArray(), StandardCharsets.UTF_8); + return new String(result.toByteArray(), UTF8); } public static String unescape(String source) { - byte bytes[] = source.getBytes(StandardCharsets.UTF_8); + byte bytes[] = source.getBytes(UTF8); ByteArrayOutputStream result = new ByteArrayOutputStream(); for (int i=0; i<bytes.length; ++i) { if (bytes[i] != '\\') { @@ -118,7 +120,7 @@ public class StringUtilities { result.write((byte) Integer.parseInt(hexdigits, 16)); i += 3; } - return new String(result.toByteArray(), StandardCharsets.UTF_8); + return new String(result.toByteArray(), UTF8); } /** diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index e882353469e..b81f0447d04 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; import java.io.IOException; @@ -31,6 +31,11 @@ public final class Utf8 { return StandardCharsets.UTF_8; } + /** To be used instead of String.String(byte[] bytes) */ + public static String toStringStd(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + /** * Utility method as toString(byte[]). * @@ -61,6 +66,13 @@ public final class Utf8 { } /** + * Uses String.getBytes directly. + */ + public static byte[] toBytesStd(String str) { + return str.getBytes(StandardCharsets.UTF_8); + } + + /** * Encode a long as its decimal representation, i.e. toAsciiBytes(15L) will * return "15" encoded as UTF-8. In other words it is an optimized version * of String.valueOf() followed by UTF-8 encoding. Avoid going through diff --git a/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java index 77967bb08fa..c6278133d4c 100644 --- a/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java @@ -1,23 +1,23 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; -import com.yahoo.text.Utf8; -import com.yahoo.text.Utf8Array; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.IdentityHashMap; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import org.junit.Test; /** * Test the ByteWriter class. ByteWriter is also implicitly tested in @@ -27,7 +27,7 @@ import static org.junit.Assert.fail; */ public class ByteWriterTestCase { - private final CharsetEncoder encoder = StandardCharsets.UTF_8.newEncoder(); + private final CharsetEncoder encoder = Charset.forName("UTF-8").newEncoder(); private final ByteArrayOutputStream stream = new ByteArrayOutputStream(); /** @@ -281,7 +281,7 @@ public class ByteWriterTestCase { @Test public void testUnMappableCharacter() throws java.io.IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); - ByteWriter writer = new ByteWriter(stream, StandardCharsets.US_ASCII.newEncoder()); + ByteWriter writer = new ByteWriter(stream, Charset.forName("ascii").newEncoder()); writer.write("yahoo\u9999bahoo"); writer.close(); assertTrue(stream.toString("ascii").contains("yahoo")); diff --git a/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java index 4769562db89..1acd225b18a 100644 --- a/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; import org.junit.Test; @@ -6,7 +6,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.io.StringReader; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -16,15 +16,15 @@ public class FileReadTestCase { @Test public void testReadByteArray() throws IOException { byte[] thisFile = IOUtils.readFileBytes(new File("src/test/java/com/yahoo/io/FileReadTestCase.java")); - String str = new String(thisFile, StandardCharsets.US_ASCII); - assertTrue(str.startsWith("// Copyright Verizon Media.")); + String str = new String(thisFile, Charset.forName("US-ASCII")); + assertTrue(str.startsWith("// Copyright 2017 Yahoo Holdings.")); assertTrue(str.endsWith("// Yeppers\n")); } @Test public void testReadString() throws IOException { String str = IOUtils.readFile(new File("src/test/java/com/yahoo/io/FileReadTestCase.java")); - assertTrue(str.startsWith("// Copyright Verizon Media.")); + assertTrue(str.startsWith("// Copyright 2017 Yahoo Holdings.")); assertTrue(str.endsWith("// Yeppers\n")); } diff --git a/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java index d250338b8fe..dcea8101d9c 100644 --- a/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java @@ -1,19 +1,28 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; import org.junit.Test; -import java.nio.charset.StandardCharsets; +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; /** * @author Simon Thoresen Hult - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class HexDumpTestCase { + private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final Charset UTF16 = Charset.forName("UTF-16"); + @Test public void requireThatToHexStringAcceptsNull() { assertNull(HexDump.toHexString(null)); @@ -22,10 +31,10 @@ public class HexDumpTestCase { @Test public void requireThatToHexStringIsUnformatted() { assertEquals("6162636465666768696A6B6C6D6E6F707172737475767778797A", - HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_8))); + HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF8))); assertEquals("FEFF006100620063006400650066006700680069006A006B006C00" + "6D006E006F0070007100720073007400750076007700780079007A", - HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_16))); + HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF16))); } } diff --git a/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java index 7f374441a18..7726cd7ea68 100644 --- a/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java @@ -1,20 +1,20 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.*; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.zip.Deflater; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.text.Utf8; /** * Check decompressor used among other things for packed summary fields. * - * @author Steinar Knutsen + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class SlowInflateTestCase { @@ -27,7 +27,7 @@ public class SlowInflateTestCase { @Before public void setUp() throws Exception { value = "000000000000000000000000000000000000000000000000000000000000000"; - raw = value.getBytes(StandardCharsets.UTF_8); + raw = Utf8.toBytesStd(value); output = new byte[raw.length * 2]; Deflater compresser = new Deflater(); compresser.setInput(raw); diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java index 3b31c2b559e..44aa4ab2ff7 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -1,7 +1,8 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; import com.yahoo.text.Utf8; +import org.junit.Ignore; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -200,7 +201,7 @@ public class JsonFormatTestCase { public void testDecodeUnicodeAmp() { final String json = "{\"body\":\"some text\\u0026more text\"}"; Slime slime = new Slime(); - new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8)); + new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); Cursor a = slime.get().field("body"); assertThat(a.asString(), is("some text&more text")); } @@ -223,7 +224,7 @@ public class JsonFormatTestCase { " }\n"; Slime slime = new Slime(); - slime = new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8)); + slime = new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); Cursor a = slime.get().field("rules"); assertThat(a.asString(), is(str)); } @@ -260,7 +261,7 @@ public class JsonFormatTestCase { private void verifyEncodeDecode(String json, boolean compact) { try { Slime slime = new Slime(); - new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8)); + new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(compact).encode(a, slime); assertEquals(json, Utf8.toString(a.toByteArray())); diff --git a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java index 8293554140d..97d0717dc0b 100644 --- a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; import com.google.common.collect.ImmutableMap; @@ -24,8 +24,8 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** - * @author Bjorn Borud - * @author Steinar Knutsen + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class Utf8TestCase { @@ -322,7 +322,7 @@ public class Utf8TestCase { for (char c=0; c < i; c++) { sb.append(c); } - assertTrue(Arrays.equals(sb.toString().getBytes(StandardCharsets.UTF_8), Utf8.toBytes(sb.toString()))); + assertTrue(Arrays.equals(Utf8.toBytesStd(sb.toString()), Utf8.toBytes(sb.toString()))); } } |