diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-01-25 21:49:20 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-25 21:49:20 +0100 |
commit | 8096d5ec0490ae7da7d3208c84546d2798781e52 (patch) | |
tree | 74c7a7d4f2e9becb6022b2119fd3719e4ced3b91 | |
parent | a0deac28d2261734f6c3be07c4a4041b0f8d52af (diff) | |
parent | ea22b6c00df28ef879ce3c1e165c9502109af65d (diff) |
Merge pull request #25734 from vespa-engine/balder/stack-2-deque-2
Balder/stack 2 deque 2
11 files changed, 146 insertions, 1008 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java index 0399664faf2..bb864fa1708 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java @@ -10,12 +10,13 @@ import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Stack; /** * Tool to verify that configs across multiple config servers are the same. @@ -42,17 +43,16 @@ public class ConfigVerification { } } - private static Map<String, Stack<String>> listConfigs(List<String> urls, CloseableHttpClient httpClient) throws IOException { + private static Map<String, Deque<String>> listConfigs(List<String> urls, CloseableHttpClient httpClient) throws IOException { Map<String, String> outputs = performRequests(urls, httpClient); - Map<String, Stack<String>> recurseMappings = new LinkedHashMap<>(); + Map<String, Deque<String>> recurseMappings = new LinkedHashMap<>(); for (Map.Entry<String, String> entry : outputs.entrySet()) { Slime slime = SlimeUtils.jsonToSlime(entry.getValue()); final List<String> list = new ArrayList<>(); slime.get().field("configs").traverse((ArrayTraverser) (idx, inspector) -> list.add(inspector.asString())); - Stack<String> stack = new Stack<>(); Collections.sort(list); - stack.addAll(list); + Deque<String> stack = new ArrayDeque<>(list); recurseMappings.put(entry.getKey(), stack); } return recurseMappings; @@ -66,10 +66,10 @@ public class ConfigVerification { return outputs; } - private static int compareConfigs(Map<String, Stack<String>> mappings, CloseableHttpClient httpClient) throws IOException { + private static int compareConfigs(Map<String, Deque<String>> mappings, CloseableHttpClient httpClient) throws IOException { for (int n = 0; n < mappings.values().iterator().next().size(); n++) { List<String> recurseUrls = new ArrayList<>(); - for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { + for (Map.Entry<String, Deque<String>> entry : mappings.entrySet()) { recurseUrls.add(entry.getValue().pop()); } if ( ! equalOutputs(performRequests(recurseUrls, httpClient))) diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java deleted file mode 100644 index 16a3ef3371d..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.config.codegen.CNode; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.config.codegen.LeafCNode; -import com.yahoo.slime.*; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.util.ConfigUtils; - -import java.io.*; -import java.util.Stack; - -/** - * @author Ulf Lilleengen - */ -public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { - - private final InnerCNode root; - private DataOutputStream out = null; - private Stack<Node> nodeStack; - - public ConfigFileFormat(InnerCNode root) { - this.root = root; - this.nodeStack = new Stack<>(); - } - - private void printPrefix() throws IOException { - for (Node node : nodeStack) { - CNode cnode = node.node; - if (cnode != root) { - encodeString(cnode.getName()); - if (cnode.isArray) { - encodeString("[" + node.arrayIndex + "]"); - if (!(cnode instanceof LeafCNode)) { - encodeString("."); - } - } else if (cnode.isMap) { - encodeString("{\"" + node.mapKey + "\"}"); - if (!(cnode instanceof LeafCNode)) { - encodeString("."); - } - } else if (cnode instanceof LeafCNode) { - encodeString(""); - } else { - encodeString("."); - } - } - } - encodeString(" "); - } - - private void encode(Inspector inspector, CNode node) throws IOException { - switch (inspector.type()) { - case BOOL: - encodeValue(String.valueOf(inspector.asBool()), (LeafCNode) node); - return; - case LONG: - encodeValue(String.valueOf(inspector.asLong()), (LeafCNode) node); - return; - case DOUBLE: - encodeValue(String.valueOf(inspector.asDouble()), (LeafCNode) node); - return; - case STRING: - encodeValue(inspector.asString(), (LeafCNode) node); - return; - case ARRAY: - encodeArray(inspector, node); - return; - case OBJECT: - if (node.isMap) { - encodeMap(inspector, node); - } else { - encodeObject(inspector, node); - } - return; - case NIX: - case DATA: - throw new IllegalArgumentException("Illegal config format supplied. Unknown type for field '" + node.getName() + "'"); - } - throw new RuntimeException("Should not be reached"); - } - - private void encodeMap(Inspector inspector, final CNode node) { - inspector.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - try { - nodeStack.push(new Node(node, -1, name)); - if (inspector.type().equals(Type.OBJECT)) { - encodeObject(inspector, node); - } else { - encode(inspector, node); - } - nodeStack.pop(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } - - private void encodeArray(Inspector inspector, final CNode node) { - inspector.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - try { - nodeStack.push(new Node(node, idx, "")); - encode(inspector, node); - nodeStack.pop(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - } - - private void encodeObject(Inspector inspector, CNode node) { - if (!node.isArray && !node.isMap) { - nodeStack.push(new Node(node)); - inspector.traverse(this); - nodeStack.pop(); - } else { - inspector.traverse(this); - } - } - - private void encodeValue(String value, LeafCNode node) throws IOException { - printPrefix(); - try { - if (node instanceof LeafCNode.StringLeaf) { - encodeStringQuoted(value); - } else if (node instanceof LeafCNode.IntegerLeaf) { - //Integer.parseInt(value); - encodeString(value); - } else if (node instanceof LeafCNode.LongLeaf) { - //Long.parseLong(value); - encodeString(value); - } else if (node instanceof LeafCNode.DoubleLeaf) { - //Double.parseDouble(value); - encodeString(value); - } else if (node instanceof LeafCNode.BooleanLeaf) { - encodeString(String.valueOf(Boolean.parseBoolean(value))); - } else if (node instanceof LeafCNode.EnumLeaf) { - // LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) node; - // TODO: Reenable this when we can return illegal config id. - // checkLegalEnumValue(enumNode, value); - encodeString(value); - } else { - encodeStringQuoted(value); - } - encodeString("\n"); - } catch (Exception e) { - throw new IllegalArgumentException("Unable to serialize field '" + node.getFullName() + "': ", e); - } - } - - private void checkLegalEnumValue(LeafCNode.EnumLeaf enumNode, String value) { - boolean found = false; - for (String legalVal : enumNode.getLegalValues()) { - if (legalVal.equals(value)) { - found = true; - } - } - if (!found) - throw new IllegalArgumentException("Illegal enum value '" + value + "'"); - } - - private void encodeStringQuoted(String s) throws IOException { - encodeString("\"" + escapeString(s) + "\""); - } - - private String escapeString(String s) { - return ConfigUtils.escapeConfigFormatValue(s); - } - - private void encodeString(String s) throws IOException { - out.write(Utf8.toBytes(s)); - } - - @Override - public void encode(OutputStream os, Slime slime) throws IOException { - encode(os, slime.get()); - } - - private void encode(OutputStream os, Inspector inspector) throws IOException { - this.out = new DataOutputStream(os); - this.nodeStack = new Stack<>(); - nodeStack.push(new Node(root)); - encode(inspector, root); - } - - @Override - public void field(String fieldName, Inspector inspector) { - try { - Node parent = nodeStack.peek(); - CNode child = parent.node.getChild(fieldName); - if (child == null) { - return; // Skip this field to optimistic - } - if (!child.isArray && !child.isMap && child instanceof LeafCNode) { - nodeStack.push(new Node(child)); - encode(inspector, child); - nodeStack.pop(); - } else { - encode(inspector, child); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private class Node { - final int arrayIndex; - final String mapKey; - final CNode node; - Node(CNode node, int arrayIndex, String mapKey) { - this.node = node; - this.arrayIndex = arrayIndex; - this.mapKey = mapKey; - } - - Node(CNode node) { - this(node, -1, ""); - } - } -} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 3705c167960..e86b13b2c98 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -8,7 +8,6 @@ import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import java.io.File; import java.lang.reflect.Constructor; @@ -17,11 +16,12 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.Stack; import java.util.logging.Logger; import static java.util.logging.Level.INFO; @@ -42,7 +42,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; private final UrlDownloader urlDownloader; - private final Stack<NamedBuilder> stack = new Stack<>(); + private final Deque<NamedBuilder> stack = new ArrayDeque<>(); public ConfigPayloadApplier(T builder) { this(builder, new IdentityPathAcquirer(), null); @@ -67,27 +67,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private void handleValue(Inspector inspector) { switch (inspector.type()) { - case NIX: - case BOOL: - case LONG: - case DOUBLE: - case STRING: - case DATA: - handleLeafValue(inspector); - break; - case ARRAY: - handleARRAY(inspector); - break; - case OBJECT: - handleOBJECT(inspector); - break; - default: + case NIX, BOOL, LONG, DOUBLE, STRING, DATA -> handleLeafValue(inspector); + case ARRAY -> handleARRAY(inspector); + case OBJECT -> handleOBJECT(inspector); + default -> { assert false : "Should not be reached"; + } } } private void handleARRAY(Inspector inspector) { - inspector.traverse((ArrayTraverser)(int index, Inspector value) -> handleArrayEntry(index, value)); + inspector.traverse((ArrayTraverser) this::handleArrayEntry); } private void handleArrayEntry(int idx, Inspector inspector) { @@ -108,11 +98,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } private void handleOBJECT(Inspector inspector) { - inspector.traverse((String name, Inspector value) -> handleObjectEntry(name, value)); + inspector.traverse(this::handleObjectEntry); NamedBuilder builder = stack.pop(); // Need to set e.g struct(Struct.Builder) here - if ( ! stack.empty()) { + if ( ! stack.isEmpty()) { try { invokeSetter(stack.peek().builder, builder.peekName(), builder.builder); } catch (Exception e) { @@ -165,7 +155,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { throw new RuntimeException("Missing map builder (this should never happen): " + stack.peek()); setMapLeafValue(name, builder.builder()); stack.push(builder); - inspector.traverse((ObjectTraverser) (key, value) -> handleObjectEntry(key, value)); + inspector.traverse(this::handleObjectEntry); stack.pop(); } @@ -296,18 +286,24 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private Object getValueFromInspector(Inspector inspector) { switch (inspector.type()) { - case STRING: + case STRING -> { return inspector.asString(); - case LONG: + } + case LONG -> { return String.valueOf(inspector.asLong()); - case DOUBLE: + } + case DOUBLE -> { return String.valueOf(inspector.asDouble()); - case NIX: + } + case NIX -> { return null; - case BOOL: + } + case BOOL -> { return String.valueOf(inspector.asBool()); - case DATA: + } + case DATA -> { return String.valueOf(inspector.asData()); + } } throw new IllegalArgumentException("Unhandled type " + inspector.type()); } @@ -372,7 +368,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return name.substring(0, 1).toUpperCase() + name.substring(1); } - private Constructor<?> lookupBuilderForStruct(String structName, String name, Class<?> currentClass) { + private Constructor<?> lookupBuilderForStruct(String structName, Class<?> currentClass) { String currentClassName = currentClass.getName(); Class<?> structClass = getInnerClass(currentClass, currentClassName + "$" + structName); if (structClass == null) { @@ -420,7 +416,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { String key = constructorCacheKey(structName, name, currentClass); Constructor<?> constructor = constructorCache.get(key); if (constructor == null) { - constructor = lookupBuilderForStruct(structName, name, currentClass); + constructor = lookupBuilderForStruct(structName, currentClass); if (constructor == null) return null; constructorCache.put(key, constructor); } @@ -437,7 +433,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private static class NamedBuilder { private final ConfigBuilder builder; - private final Stack<String> names = new Stack<>(); // if empty, the builder is the root builder + private final Deque<String> names = new ArrayDeque<>(); // if empty, the builder is the root builder NamedBuilder(ConfigBuilder builder) { this.builder = builder; @@ -456,7 +452,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return names.peek(); } - Stack<String> nameStack() { + Deque<String> nameStack() { return names; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java index 875033f1ffa..5193ecddc82 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -5,8 +5,9 @@ import com.yahoo.slime.Cursor; import com.yahoo.yolean.trace.TraceNode; import com.yahoo.yolean.trace.TraceVisitor; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Iterator; -import java.util.Stack; /** * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. @@ -17,7 +18,7 @@ public class SlimeTraceSerializer extends TraceVisitor { static final String TIMESTAMP = "timestamp"; static final String PAYLOAD = "payload"; static final String CHILDREN = "children"; - final Stack<Cursor> cursors = new Stack<>(); + final Deque<Cursor> cursors = new ArrayDeque<>(); public SlimeTraceSerializer(Cursor cursor) { cursors.push(cursor); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java deleted file mode 100644 index ab07b669bd0..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.foo.ArraytypesConfig; -import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.foo.StructtypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.foo.MaptypesConfig; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.text.StringUtilities; -import com.yahoo.text.Utf8; -import org.junit.Test; -import org.junit.Ignore; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringReader; -import java.nio.charset.StandardCharsets; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class ConfigFileFormatterTest { - - private final String expected_simpletypes = "stringval \"foo\"\n" + - "intval 324234\n" + - "longval 324\n" + - "doubleval 3.455\n" + - "enumval VAL2\n" + - "boolval true\n"; - - @Test - public void require_that_basic_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "foo"); - root.setString("intval", "324234"); - root.setString("longval", "324"); - root.setString("doubleval", "3.455"); - root.setString("enumval", "VAL2"); - root.setString("boolval", "true"); - - assertConfigFormat(slime, expected_simpletypes); - } - - @Test - public void require_that_basic_formatting_is_correct_with_types() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "foo"); - root.setLong("intval", 324234); - root.setLong("longval", 324); - root.setDouble("doubleval", 3.455); - root.setString("enumval", "VAL2"); - root.setBool("boolval", true); - - assertConfigFormat(slime, expected_simpletypes); - } - - private void assertConfigFormat(Slime slime, String expected_simpletypes) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals(expected_simpletypes, baos.toString()); - } - - @Test - public void require_that_field_not_found_is_ignored() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("nosuchfield", "bar"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertTrue(baos.toString().isEmpty()); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_int_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("intval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_long_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("longval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_double_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("doubleval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - @Test - public void require_that_illegal_boolean_becomes_false() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("boolval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("boolval false\n", baos.toString()); - } - - // TODO: Remove this when we can reenable typechecking. - @Test - public void require_that_types_are_not_checked() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("enumval", "null"); - root.setString("intval", "null"); - root.setString("longval", "null"); - root.setString("boolval", "null"); - root.setString("doubleval", "null"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("enumval null\nintval null\nlongval null\nboolval false\ndoubleval null\n", - baos.toString(StandardCharsets.UTF_8)); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_enum_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("enumval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - @Test - public void require_that_strings_are_encoded() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - String value = "\u7d22"; - root.setString("stringval", value); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("stringval \"" + value + "\"\n", baos.toString(StandardCharsets.UTF_8)); - } - - @Test - public void require_that_array_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor boolarr = root.setArray("boolarr"); - boolarr.addString("true"); - boolarr.addString("false"); - Cursor doublearr = root.setArray("doublearr"); - doublearr.addString("3.14"); - doublearr.addString("1.414"); - Cursor enumarr = root.setArray("enumarr"); - enumarr.addString("VAL1"); - enumarr.addString("VAL2"); - Cursor intarr = root.setArray("intarr"); - intarr.addString("3"); - intarr.addString("5"); - Cursor longarr = root.setArray("longarr"); - longarr.addString("55"); - longarr.addString("66"); - Cursor stringarr = root.setArray("stringarr"); - stringarr.addString("foo"); - stringarr.addString("bar"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("arraytypes", new StringReader(StringUtilities.implode(ArraytypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "boolarr[0] true\n" + - "boolarr[1] false\n" + - "doublearr[0] 3.14\n" + - "doublearr[1] 1.414\n" + - "enumarr[0] VAL1\n" + - "enumarr[1] VAL2\n" + - "intarr[0] 3\n" + - "intarr[1] 5\n" + - "longarr[0] 55\n" + - "longarr[1] 66\n" + - "stringarr[0] \"foo\"\n" + - "stringarr[1] \"bar\"\n", - baos.toString()); - } - - @Test - public void require_that_map_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor boolval = root.setObject("boolmap"); - boolval.setString("foo", "true"); - boolval.setString("bar", "false"); - root.setObject("intmap").setString("foo", "1234"); - root.setObject("longmap").setString("foo", "12345"); - root.setObject("doublemap").setString("foo", "3.14"); - root.setObject("stringmap").setString("foo", "bar"); - root.setObject("innermap").setObject("bar").setString("foo", "1234"); - root.setObject("nestedmap").setObject("baz").setObject("inner").setString("foo", "1234"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("maptypes", new StringReader(StringUtilities.implode(MaptypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "boolmap{\"foo\"} true\n" + - "boolmap{\"bar\"} false\n" + - "intmap{\"foo\"} 1234\n" + - "longmap{\"foo\"} 12345\n" + - "doublemap{\"foo\"} 3.14\n" + - "stringmap{\"foo\"} \"bar\"\n" + - "innermap{\"bar\"}.foo 1234\n" + - "nestedmap{\"baz\"}.inner{\"foo\"} 1234\n", - baos.toString()); - } - - @Test - public void require_that_struct_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor simple = root.setObject("simple"); - simple.setString("name", "myname"); - simple.setString("gender", "FEMALE"); - Cursor array = simple.setArray("emails"); - array.addString("foo@bar.com"); - array.addString("bar@baz.net"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "simple.name \"myname\"\n" + - "simple.gender FEMALE\n" + - "simple.emails[0] \"foo@bar.com\"\n" + - "simple.emails[1] \"bar@baz.net\"\n", - baos.toString()); - } - - @Test - public void require_that_complex_struct_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - Cursor nested = root.setObject("nested"); - Cursor nested_inner = nested.setObject("inner"); - nested_inner.setString("name", "baz"); - nested_inner.setString("gender", "FEMALE"); - Cursor nested_inner_arr = nested_inner.setArray("emails"); - nested_inner_arr.addString("foo"); - nested_inner_arr.addString("bar"); - - Cursor nestedarr = root.setArray("nestedarr"); - Cursor nestedarr1 = nestedarr.addObject(); - Cursor inner1 = nestedarr1.setObject("inner"); - inner1.setString("name", "foo"); - inner1.setString("gender", "FEMALE"); - Cursor inner1arr = inner1.setArray("emails"); - inner1arr.addString("foo@bar"); - inner1arr.addString("bar@foo"); - - Cursor complexarr = root.setArray("complexarr"); - Cursor complexarr1 = complexarr.addObject(); - Cursor innerarr1 = complexarr1.setArray("innerarr"); - Cursor innerarr11 = innerarr1.addObject(); - innerarr11.setString("name", "bar"); - innerarr11.setString("gender", "MALE"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("nested.inner.name \"baz\"\n" + - "nested.inner.gender FEMALE\n" + - "nested.inner.emails[0] \"foo\"\n" + - "nested.inner.emails[1] \"bar\"\n" + - "nestedarr[0].inner.name \"foo\"\n" + - "nestedarr[0].inner.gender FEMALE\n" + - "nestedarr[0].inner.emails[0] \"foo@bar\"\n" + - "nestedarr[0].inner.emails[1] \"bar@foo\"\n" + - "complexarr[0].innerarr[0].name \"bar\"\n" + - "complexarr[0].innerarr[0].gender MALE\n", - baos.toString()); - } - - @Test - public void require_that_strings_are_properly_escaped() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "some\"quotes\\\"instring"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("stringval \"some\\\"quotes\\\\\\\"instring\"\n", baos.toString()); - } - - @Test - @Ignore - public void require_that_utf8_works() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - final String input = "Hei \u00E6\u00F8\u00E5 \n \uBC14\uB451 \u00C6\u00D8\u00C5 hallo"; - root.setString("stringval", input); - System.out.println(bytesToHexString(Utf8.toBytes(input))); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - System.out.println(bytesToHexString(baos.toByteArray())); - assertEquals(Utf8.toString(baos.toByteArray()), "stringval \"" + input + "\"\n"); - } - - private static String bytesToHexString(byte[] bytes){ - StringBuilder sb = new StringBuilder(); - for(byte b : bytes){ - sb.append(String.format("%02x", b&0xff)); - } - return sb.toString(); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java index 60a9dbc96db..4378c4cbd47 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java @@ -5,9 +5,10 @@ import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Range; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.HashSet; import java.util.Set; -import java.util.Stack; /** * Allocates port ranges for all configserver tests. @@ -28,7 +29,7 @@ public class PortRangeAllocator { private static class PortRange { private final Set<Integer> takenPorts = new HashSet<>(); - private final Stack<Integer> freePorts = new Stack<>(); + private final Deque<Integer> freePorts = new ArrayDeque<>(); private static final int first = 18651; private static final int last = 18899; // see: factory/doc/port-ranges diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 1da16b4d277..efb02034db9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -7,7 +7,14 @@ import com.yahoo.prelude.query.TermType; import com.yahoo.prelude.semantics.rule.Condition; import com.yahoo.prelude.semantics.rule.ProductionRule; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + /** * A particular evaluation of a particular rule. @@ -41,15 +48,15 @@ public class RuleEvaluation { private String currentContext; /** A list of referencedMatches */ - private final List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); + private final List<ReferencedMatches> referencedMatchesList = new ArrayList<>(); - private final List<Match> nonreferencedMatches = new java.util.ArrayList<>(); + private final List<Match> nonreferencedMatches = new ArrayList<>(); /** The evaluation owning this */ private final Evaluation evaluation; /** The choice points saved in this evaluation */ - private Stack<Choicepoint> choicepoints = null; + private Deque<Choicepoint> choicepoints = null; /* The last value returned by a condition evaluated in this, may be null */ private Object value = null; @@ -98,20 +105,18 @@ public class RuleEvaluation { int calculateMatchDigest(ProductionRule rule) { int matchDigest = rule.hashCode(); int matchCounter = 1; - for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches = i.next(); + for (ReferencedMatches matches : referencedMatchesList) { int termCounter = 0; for (Iterator<Match> j = matches.matchIterator(); j.hasNext(); ) { Match match = j.next(); - matchDigest = 7 * matchDigest * matchCounter+ - 71 * termCounter + - match.hashCode(); + matchDigest = 7 * matchDigest * matchCounter + + 71 * termCounter + + match.hashCode(); termCounter++; } matchCounter++; } - for (Iterator<Match> i = nonreferencedMatches.iterator(); i.hasNext(); ) { - Match match = i.next(); + for (Match match : nonreferencedMatches) { matchDigest = 7 * matchDigest * matchCounter + match.hashCode(); matchCounter++; } @@ -163,10 +168,8 @@ public class RuleEvaluation { public int getPosition() { return position; } /** Sets a new current label and returns the previous one */ - public String setCurrentLabel(String currentLabel) { - String oldLabel = currentLabel; + public void setCurrentLabel(String currentLabel) { this.currentLabel = currentLabel; - return oldLabel; } public String getCurrentLabel() { return currentLabel; } @@ -226,8 +229,7 @@ public class RuleEvaluation { /** Returns the referenced matches for a context name, or null if none */ public ReferencedMatches getReferencedMatches(String name) { - for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches = i.next(); + for (ReferencedMatches matches : referencedMatchesList) { if (name.equals(matches.getContextName())) return matches; } @@ -307,7 +309,7 @@ public class RuleEvaluation { public Choicepoint getChoicepoint(Condition condition, boolean create) { if (choicepoints == null) { if ( ! create) return null; - choicepoints = new java.util.Stack<>(); + choicepoints = new ArrayDeque<>(); } Choicepoint choicepoint=lookupChoicepoint(condition); if (choicepoint == null) { @@ -319,8 +321,7 @@ public class RuleEvaluation { } private Choicepoint lookupChoicepoint(Condition condition) { - for (Iterator<Choicepoint> i = choicepoints.iterator(); i.hasNext(); ) { - Choicepoint choicepoint = i.next(); + for (Choicepoint choicepoint : choicepoints) { if (condition == choicepoint.getCondition()) return choicepoint; } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java index 03cd9b48de2..a50e3cd6c69 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java @@ -6,11 +6,12 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.Vector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -287,7 +288,7 @@ public class VisitorIteratorTestCase { // keeping some in the active set and some in pending int pendingTotal = buckets / 8; int activeTotal = buckets / 8; - Vector<VisitorIterator.BucketProgress> trackedBuckets = new Vector<VisitorIterator.BucketProgress>(); + List<VisitorIterator.BucketProgress> trackedBuckets = new ArrayList<>(); // Pre-fetch, since otherwise we'd reuse pending buckets for (int i = 0; i < pendingTotal + activeTotal; ++i) { @@ -760,8 +761,8 @@ public class VisitorIteratorTestCase { assertTrue(bk4.compareTo(bk3) > 0); ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL); ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L); - assertTrue(bk5.compareTo(bk2) == 0); - assertTrue(bk6.compareTo(bk3) == 0); + assertEquals(0, bk5.compareTo(bk2)); + assertEquals(0, bk6.compareTo(bk3)); } private void doTestBucketKeyGeneration(int db) { @@ -1384,16 +1385,17 @@ public class VisitorIteratorTestCase { @Test public void testImportProgressWithOutdatedDistribution() throws ParseException { - String input = "VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"; - - int db = 12; + String input = """ + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """; + BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); assertEquals(10, p.getDistributionBitCount()); @@ -1426,14 +1428,15 @@ public class VisitorIteratorTestCase { public void testImportInconsistentProgressIncrease() throws ParseException { // Bucket progress "file" that upon time of changing from 4 to 7 // distribution bits and writing the progress had an active bucket - String input = "VDS bucket progress file\n" + - "7\n" + - "32\n" + - "24\n" + - "128\n" + - "100000000000000c:0\n"; + String input = """ + VDS bucket progress file + 7 + 32 + 24 + 128 + 100000000000000c:0 + """; // Now we're at 8 distribution bits - int db = 8; BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); assertEquals(7, p.getDistributionBitCount()); @@ -1472,12 +1475,14 @@ public class VisitorIteratorTestCase { public void testImportInconsistentProgressDecrease() throws ParseException { // Bucket progress "file" that upon time of changing from 4 to 7 // distribution bits and writing the progress had an active bucket - String input = "VDS bucket progress file\n" + - "7\n" + - "32\n" + - "24\n" + - "128\n" + - "100000000000000c:0\n"; + String input = """ + VDS bucket progress file + 7 + 32 + 24 + 128 + 100000000000000c:0 + """; BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); @@ -1553,8 +1558,15 @@ public class VisitorIteratorTestCase { // Try to pass a known document selection to an unknown docsel iterator boolean caughtIt = false; try { - ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n" - + "8000000000001f49:0\n8000000000001a85:0\n"); + ProgressToken p = new ProgressToken(""" + VDS bucket progress file + 16 + 3 + 1 + 3 + 8000000000001f49:0 + 8000000000001a85:0 + """); VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p); } @@ -1566,14 +1578,16 @@ public class VisitorIteratorTestCase { // Now try it the other way around caughtIt = false; try { - ProgressToken p = new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"); + ProgressToken p = new ProgressToken(""" + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """); VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p); } @@ -1645,13 +1659,14 @@ public class VisitorIteratorTestCase { public void testMalformedProgressFile() { boolean caughtIt = false; try { - new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:"); + new ProgressToken(""" + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:"""); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1662,9 +1677,11 @@ public class VisitorIteratorTestCase { public void testFailOnTooFewLinesInFile() { boolean caughtIt = false; try { - new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n"); + new ProgressToken(""" + VDS bucket progress file + 10 + 503 + """); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1675,13 +1692,14 @@ public class VisitorIteratorTestCase { public void testUnknownFirstHeaderLine() { boolean caughtIt = false; try { - new ProgressToken("Smurf Time 3000\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0"); + new ProgressToken(""" + Smurf Time 3000 + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0"""); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1690,14 +1708,16 @@ public class VisitorIteratorTestCase { @Test public void testBinaryProgressSerialization() { - String input = "VDS bucket progress file (48.828125% completed)\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"; + String input = """ + VDS bucket progress file (48.828125% completed) + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """; ProgressToken p = new ProgressToken(input); byte[] buf = p.serialize(); ProgressToken p2 = new ProgressToken(buf); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index ab702c85ffc..75da8d2c068 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -14,18 +14,18 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.Stack; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -167,7 +167,7 @@ public class FileFinder { // Only need to traverse as deep as we want to match, unless we want to match everything in directories // already matched Files.walkFileTree(basePath, Set.of(), maxDepth, new SimpleFileVisitor<>() { - private final Stack<FileAttributes> matchingDirectoryStack = new Stack<>(); + private final Deque<FileAttributes> matchingDirectoryStack = new ArrayDeque<>(); private int currentLevel = -1; @Override diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java deleted file mode 100644 index 4b7d2ba4094..00000000000 --- a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.nio.channels.WritableByteChannel; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Stack; -import java.util.LinkedList; -import java.util.Iterator; -import java.nio.ByteBuffer; - -/** - * @author Bjørn Borud - */ -public class GrowableBufferOutputStream extends OutputStream { - - private ByteBuffer lastBuffer; - private final ByteBuffer directBuffer; - private final LinkedList<ByteBuffer> bufferList = new LinkedList<>(); - private final Stack<ByteBuffer> recycledBuffers = new Stack<>(); - - private final int bufferSize; - private final int maxBuffers; - - public GrowableBufferOutputStream(int bufferSize, int maxBuffers) { - this.bufferSize = bufferSize; - this.maxBuffers = maxBuffers; - lastBuffer = ByteBuffer.allocate(bufferSize); - directBuffer = ByteBuffer.allocateDirect(bufferSize); - } - - @Override - public void write(byte[] cbuf, int off, int len) throws IOException { - if (lastBuffer.remaining() >= len) { - lastBuffer.put(cbuf, off, len); - return; - } - - int residue = len; - - while (residue > 0) { - int newOffset = len - residue; - int toWrite = Math.min(lastBuffer.remaining(), residue); - - lastBuffer.put(cbuf, newOffset, toWrite); - residue -= toWrite; - if (residue != 0) { - extend(); - } - } - } - - @Override - public void write(byte[] b) throws IOException { - write(b,0,b.length); - } - - @Override - public String toString() { - return "GrowableBufferOutputStream, writable size " + writableSize() - + " bytes, " + numWritableBuffers() + " buffers, last buffer" - + " position " + lastBuffer.position() + ", last buffer limit " - + lastBuffer.limit(); - } - - public void write(int b) { - if (lastBuffer.remaining() == 0) { - extend(); - } - lastBuffer.put((byte) b); - } - - @Override - public void flush() { - // if the last buffer is untouched we do not need to do anything; if - // it has been touched we call extend(), which enqueues the buffer - // and allocates or recycles a buffer for us - if (lastBuffer.position() > 0) { - extend(); - } - } - - @Override - public void close() { - flush(); - } - - public int channelWrite(WritableByteChannel channel) throws IOException { - ByteBuffer buffer; - int totalWritten = 0; - - while (!bufferList.isEmpty()) { - buffer = bufferList.getFirst(); - int written = 0; - - synchronized (directBuffer) { - directBuffer.clear(); - directBuffer.put(buffer); - directBuffer.flip(); - written = channel.write(directBuffer); - int left = directBuffer.remaining(); - - if (left > 0) { - int oldpos = buffer.position(); - - buffer.position(oldpos - left); - } - totalWritten += written; - } - - // if we've completed writing this buffer we can dispose of it - if (buffer.remaining() == 0) { - bufferList.removeFirst(); - recycleBuffer(buffer); - } - - // if we didn't write any bytes we terminate - if (written == 0) { - break; - } - } - - return totalWritten; - } - - public int numWritableBuffers() { - return bufferList.size(); - } - - public void clear() { - flush(); - bufferList.clear(); - } - - public void clearCache() { - recycledBuffers.clear(); - } - - public void clearAll() { - clear(); - clearCache(); - } - - public int writableSize() { - Iterator<ByteBuffer> it = bufferList.iterator(); - int size = 0; - - while (it.hasNext()) { - size += (it.next()).remaining(); - } - - return size; - } - - public ByteBuffer[] getWritableBuffers() { - flush(); - ByteBuffer[] result = new ByteBuffer[numWritableBuffers()]; - return bufferList.toArray(result); - } - - private void extend() { - enqueueBuffer(lastBuffer); - - if (recycledBuffers.empty()) { - lastBuffer = ByteBuffer.allocate(bufferSize); - } else { - lastBuffer = recycledBuffers.pop(); - lastBuffer.clear(); - } - } - - private void enqueueBuffer(ByteBuffer buffer) { - buffer.flip(); - bufferList.addLast(buffer); - } - - private void recycleBuffer(ByteBuffer buffer) { - if (recycledBuffers.size() >= maxBuffers) { - return; - } - recycledBuffers.push(buffer); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java b/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java deleted file mode 100644 index 9c905979131..00000000000 --- a/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.nio.channels.WritableByteChannel; -import java.nio.ByteBuffer; -import java.io.IOException; -import com.yahoo.io.GrowableBufferOutputStream; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - - -/** - * Tests the GrowableBufferOutputStream - * - * @author Bjorn Borud - */ -public class GrowableBufferOutputStreamTestCase { - - private byte[] testData; - - static class DummyWritableByteChannel implements WritableByteChannel { - private ByteBuffer buffer; - - public DummyWritableByteChannel(ByteBuffer buffer) { - this.buffer = buffer; - } - - public int write(ByteBuffer src) { - int written = Math.min(src.remaining(), buffer.remaining()); - - if (buffer.remaining() < src.remaining()) { - ByteBuffer tmp = src.slice(); - - tmp.limit(written); - src.position(src.position() + written); - } else { - buffer.put(src); - } - return written; - } - - public boolean isOpen() { - return true; - } - - public void close() {} - } - - @Before - public void setUp() { - testData = new byte[100]; - for (int i = 0; i < 100; ++i) { - testData[i] = (byte) i; - } - } - - @Test - public void testSimple() throws IOException { - GrowableBufferOutputStream g = new GrowableBufferOutputStream(10, 5); - - g.write(testData, 0, 100); - g.flush(); - assertEquals(10, g.numWritableBuffers()); - assertEquals(100, g.writableSize()); - - ByteBuffer sink = ByteBuffer.allocate(60); - DummyWritableByteChannel channel = new DummyWritableByteChannel(sink); - int written = g.channelWrite(channel); - - assertEquals(60, written); - assertEquals(60, sink.position()); - assertEquals(40, g.writableSize()); - - // there should be 4 buffers left now - assertEquals(4, g.numWritableBuffers()); - - // ensure that we got what we expected - for (int i = 0; i < 60; ++i) { - if (((int) sink.get(i)) != i) { - fail(); - } - } - - // then we write more data - g.write(testData, 0, 100); - g.flush(); - assertEquals(140, g.writableSize()); - - // ...which implies that we should now have 14 writable buffers - assertEquals(14, g.numWritableBuffers()); - - // reset the sink so it can consume more data - sink.clear(); - - // then write more to the DummyWritableByteChannel - written = g.channelWrite(channel); - assertEquals(60, written); - assertEquals(60, sink.position()); - assertEquals(80, g.writableSize()); - - // now there should be 8 buffers - assertEquals(8, g.numWritableBuffers()); - - // ensure that we got what we expected - for (int i = 0; i < 60; ++i) { - int val = (int) sink.get(i); - int expected = (i + 60) % 100; - - if (val != expected) { - fail("Value was " + val + " and not " + i); - } - } - - // when we clear there should be no buffers - g.clear(); - assertEquals(0, g.numWritableBuffers()); - assertEquals(0, g.writableSize()); - - // ditto after flush after clear - g.flush(); - assertEquals(0, g.numWritableBuffers()); - - // flush the cache too - g.clearAll(); - assertEquals(0, g.numWritableBuffers()); - } - -} |