Revert "Remove unused Utf8 methods"

11 files changed, 85 insertions, 56 deletions

diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index 56c5d1b5081..7e7e376a8df 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -3061,8 +3061,10 @@
     "methods": [
       "public void <init>()",
       "public static java.nio.charset.Charset getCharset()",
+      "public static java.lang.String toStringStd(byte[])",
       "public static java.lang.String toString(byte[], int, int)",
       "public static java.lang.String toString(java.nio.ByteBuffer)",
+      "public static byte[] toBytesStd(java.lang.String)",
       "public static byte[] toAsciiBytes(long)",
       "public static byte[] toAsciiBytes(boolean)",
       "public static byte[] toBytes(java.lang.String)",
diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
index 266b7857363..f2de0ace476 100644
--- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
+++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java
@@ -1,24 +1,26 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
 
 import java.io.*;
 import java.nio.channels.FileChannel;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.util.List;
 import java.nio.charset.Charset;
 import java.nio.ByteBuffer;
 
+
 /**
- * Some static I/O convenience methods.
+ * <p>Some static io convenience methods.</p>
  *
- * @author bratseth
- * @author Bjorn Borud
+ * @author  bratseth
+ * @author  Bjorn Borud
  */
 public abstract class IOUtils {
 
+    static private final Charset utf8Charset = Charset.forName("utf-8");
+
     /** Closes a writer, or does nothing if the writer is null */
     public static void closeWriter(Writer writer) {
         if (writer == null) return;
@@ -327,7 +329,7 @@ public abstract class IOUtils {
      * Encodes string as UTF-8 into ByteBuffer
      */
     public static ByteBuffer utf8ByteBuffer(String s) {
-        return StandardCharsets.UTF_8.encode(s);
+        return utf8Charset.encode(s);
     }
 
     /**
@@ -339,7 +341,7 @@ public abstract class IOUtils {
     public static String readFile(File file) throws IOException {
         try {
             if (file == null) return null;
-            return Files.readString(file.toPath());
+            return new String(Files.readAllBytes(file.toPath()), "utf-8");
         }
         catch (NoSuchFileException e) {
             throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'");
diff --git a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java
index f8a820a7421..6e7f9f5e50e 100644
--- a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java
+++ b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java
@@ -1,9 +1,10 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
 import java.nio.Buffer;
+import java.nio.charset.Charset;
 import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
+
 
 /**
  * Conventient utility for reading lines from ByteBuffers.  Please
@@ -11,10 +12,11 @@ import java.nio.charset.StandardCharsets;
  * ByteBuffer abstraction is somewhat clumsy and thus usage of this
  * code requires that you understand the semantics clearly.
  *
- * @author Bjorn Borud
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
  *
  */
 public class ReadLine {
+    static private Charset charset = Charset.forName("latin1");
 
     /**
      * Extract next line from a byte buffer.  Looks for EOL characters
@@ -47,7 +49,7 @@ public class ReadLine {
                 // The downcast to Buffer is done to avoid "redundant cast" warning on Java 9.
                 // TODO: when Java 8 is gone, remove the casts and above comments.
                 // extract string between start and i.
-                String line = StandardCharsets.ISO_8859_1.decode((ByteBuffer) ((Buffer)buffer.slice()).limit(i - start)).toString();
+                String line = charset.decode((ByteBuffer) ((Buffer)buffer.slice()).limit(i - start)).toString();
 
                 // skip remaining
                 for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) {
@@ -78,5 +80,4 @@ public class ReadLine {
     static boolean isEolChar(byte b) {
         return ((10 == b) || (13 == b));
     }
-
 }
diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
index 74458be4584..370d079b3ec 100644
--- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
+++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
@@ -1,9 +1,9 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.text;
 
 import com.google.common.collect.ImmutableSet;
 
-import java.nio.charset.StandardCharsets;
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.List;
 import java.io.ByteArrayOutputStream;
@@ -19,6 +19,8 @@ import java.util.Set;
 // TODO: Text utilities should which are still needed should move to Text. This should be deprecated.
 public class StringUtilities {
 
+    private static Charset UTF8 = Charset.forName("utf8");
+
     private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); }
 
     private static class ReplacementCharacters {
@@ -63,7 +65,7 @@ public class StringUtilities {
      * @return The escaped string
      */
     public static String escape(String source, char delimiter) {
-        byte bytes[] = source.getBytes(StandardCharsets.UTF_8);
+        byte bytes[] = source.getBytes(UTF8);
         ByteArrayOutputStream result = new ByteArrayOutputStream();
         for (byte b : bytes) {
             int val = b;
@@ -84,11 +86,11 @@ public class StringUtilities {
                 result.write(replacementCharacters.replacement2[val]);
             }
         }
-        return new String(result.toByteArray(), StandardCharsets.UTF_8);
+        return new String(result.toByteArray(), UTF8);
     }
 
     public static String unescape(String source) {
-        byte bytes[] = source.getBytes(StandardCharsets.UTF_8);
+        byte bytes[] = source.getBytes(UTF8);
         ByteArrayOutputStream result = new ByteArrayOutputStream();
         for (int i=0; i<bytes.length; ++i) {
             if (bytes[i] != '\\') {
@@ -118,7 +120,7 @@ public class StringUtilities {
             result.write((byte) Integer.parseInt(hexdigits, 16));
             i += 3;
         }
-        return new String(result.toByteArray(), StandardCharsets.UTF_8);
+        return new String(result.toByteArray(), UTF8);
     }
 
     /**
diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java
index e882353469e..b81f0447d04 100644
--- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java
+++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java
@@ -1,4 +1,4 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.text;
 
 import java.io.IOException;
@@ -31,6 +31,11 @@ public final class Utf8 {
         return StandardCharsets.UTF_8;
     }
 
+    /** To be used instead of String.String(byte[] bytes) */
+    public static String toStringStd(byte[] data) {
+        return new String(data, StandardCharsets.UTF_8);
+    }
+
     /**
      * Utility method as toString(byte[]).
      *
@@ -61,6 +66,13 @@ public final class Utf8 {
     }
 
     /**
+     * Uses String.getBytes directly.
+     */
+    public static byte[] toBytesStd(String str) {
+        return str.getBytes(StandardCharsets.UTF_8);
+    }
+
+    /**
      * Encode a long as its decimal representation, i.e. toAsciiBytes(15L) will
      * return "15" encoded as UTF-8. In other words it is an optimized version
      * of String.valueOf() followed by UTF-8 encoding. Avoid going through
diff --git a/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java
index 77967bb08fa..c6278133d4c 100644
--- a/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java
@@ -1,23 +1,23 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
-import com.yahoo.text.Utf8;
-import com.yahoo.text.Utf8Array;
-import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.nio.charset.CharsetEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.IdentityHashMap;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8Array;
+import org.junit.Test;
 
 /**
  * Test the ByteWriter class. ByteWriter is also implicitly tested in
@@ -27,7 +27,7 @@ import static org.junit.Assert.fail;
  */
 public class ByteWriterTestCase {
 
-    private final CharsetEncoder encoder = StandardCharsets.UTF_8.newEncoder();
+    private final CharsetEncoder encoder = Charset.forName("UTF-8").newEncoder();
     private final ByteArrayOutputStream stream = new ByteArrayOutputStream();
 
     /**
@@ -281,7 +281,7 @@ public class ByteWriterTestCase {
     @Test
     public void testUnMappableCharacter() throws java.io.IOException {
         ByteArrayOutputStream stream = new ByteArrayOutputStream();
-        ByteWriter writer = new ByteWriter(stream, StandardCharsets.US_ASCII.newEncoder());
+        ByteWriter writer = new ByteWriter(stream, Charset.forName("ascii").newEncoder());
         writer.write("yahoo\u9999bahoo");
         writer.close();
         assertTrue(stream.toString("ascii").contains("yahoo"));
diff --git a/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java
index 4769562db89..1acd225b18a 100644
--- a/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java
@@ -1,4 +1,4 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
 import org.junit.Test;
@@ -6,7 +6,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
+import java.nio.charset.Charset;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -16,15 +16,15 @@ public class FileReadTestCase {
     @Test
     public void testReadByteArray() throws IOException {
         byte[] thisFile = IOUtils.readFileBytes(new File("src/test/java/com/yahoo/io/FileReadTestCase.java"));
-        String str = new String(thisFile, StandardCharsets.US_ASCII);
-        assertTrue(str.startsWith("// Copyright Verizon Media."));
+        String str = new String(thisFile, Charset.forName("US-ASCII"));
+        assertTrue(str.startsWith("// Copyright 2017 Yahoo Holdings."));
         assertTrue(str.endsWith("// Yeppers\n"));
     }
 
     @Test
     public void testReadString() throws IOException {
         String str = IOUtils.readFile(new File("src/test/java/com/yahoo/io/FileReadTestCase.java"));
-        assertTrue(str.startsWith("// Copyright Verizon Media."));
+        assertTrue(str.startsWith("// Copyright 2017 Yahoo Holdings."));
         assertTrue(str.endsWith("// Yeppers\n"));
     }
 
diff --git a/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java
index d250338b8fe..dcea8101d9c 100644
--- a/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java
@@ -1,19 +1,28 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
 import org.junit.Test;
 
-import java.nio.charset.StandardCharsets;
+import com.yahoo.text.Utf8;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
 /**
  * @author Simon Thoresen Hult
- * @author Steinar Knutsen
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
  */
 public class HexDumpTestCase {
 
+    private static final Charset UTF8 = Charset.forName("UTF-8");
+    private static final Charset UTF16 = Charset.forName("UTF-16");
+
     @Test
     public void requireThatToHexStringAcceptsNull() {
         assertNull(HexDump.toHexString(null));
@@ -22,10 +31,10 @@ public class HexDumpTestCase {
     @Test
     public void requireThatToHexStringIsUnformatted() {
         assertEquals("6162636465666768696A6B6C6D6E6F707172737475767778797A",
-                     HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_8)));
+                     HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF8)));
         assertEquals("FEFF006100620063006400650066006700680069006A006B006C00" +
                      "6D006E006F0070007100720073007400750076007700780079007A",
-                     HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_16)));
+                     HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF16)));
     }
 
 }
diff --git a/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java
index 7f374441a18..7726cd7ea68 100644
--- a/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java
@@ -1,20 +1,20 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.io;
 
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.*;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.zip.Deflater;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.yahoo.text.Utf8;
 
 /**
  * Check decompressor used among other things for packed summary fields.
  *
- * @author Steinar Knutsen
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
  */
 public class SlowInflateTestCase {
 
@@ -27,7 +27,7 @@ public class SlowInflateTestCase {
     @Before
     public void setUp() throws Exception {
         value = "000000000000000000000000000000000000000000000000000000000000000";
-        raw = value.getBytes(StandardCharsets.UTF_8);
+        raw = Utf8.toBytesStd(value);
         output = new byte[raw.length * 2];
         Deflater compresser = new Deflater();
         compresser.setInput(raw);
diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java
index 3b31c2b559e..44aa4ab2ff7 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java
@@ -1,7 +1,8 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.slime;
 
 import com.yahoo.text.Utf8;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
@@ -200,7 +201,7 @@ public class JsonFormatTestCase {
     public void testDecodeUnicodeAmp() {
         final String json = "{\"body\":\"some text\\u0026more text\"}";
         Slime slime = new Slime();
-        new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8));
+        new JsonDecoder().decode(slime, Utf8.toBytesStd(json));
         Cursor a = slime.get().field("body");
         assertThat(a.asString(), is("some text&more text"));
     }
@@ -223,7 +224,7 @@ public class JsonFormatTestCase {
                 "        }\n";
 
         Slime slime = new Slime();
-        slime = new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8));
+        slime = new JsonDecoder().decode(slime, Utf8.toBytesStd(json));
         Cursor a = slime.get().field("rules");
         assertThat(a.asString(), is(str));
     }
@@ -260,7 +261,7 @@ public class JsonFormatTestCase {
     private void verifyEncodeDecode(String json, boolean compact) {
         try {
             Slime slime = new Slime();
-            new JsonDecoder().decode(slime, json.getBytes(StandardCharsets.UTF_8));
+            new JsonDecoder().decode(slime, Utf8.toBytesStd(json));
             ByteArrayOutputStream a = new ByteArrayOutputStream();
             new JsonFormat(compact).encode(a, slime);
             assertEquals(json, Utf8.toString(a.toByteArray()));
diff --git a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java
index 8293554140d..97d0717dc0b 100644
--- a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java
@@ -1,4 +1,4 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.text;
 
 import com.google.common.collect.ImmutableMap;
@@ -24,8 +24,8 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
- * @author Bjorn Borud
- * @author Steinar Knutsen
+ * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
  */
 public class Utf8TestCase {
 
@@ -322,7 +322,7 @@ public class Utf8TestCase {
             for (char c=0; c < i; c++) {
                 sb.append(c);
             }
-            assertTrue(Arrays.equals(sb.toString().getBytes(StandardCharsets.UTF_8), Utf8.toBytes(sb.toString())));
+            assertTrue(Arrays.equals(Utf8.toBytesStd(sb.toString()), Utf8.toBytes(sb.toString())));
         }
     }