diff options
1045 files changed, 8863 insertions, 5213 deletions
diff --git a/.gitignore b/.gitignore index c1d500c3c4f..870db7d4a66 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ Testing /.ninja_log /build.ninja /rules.ninja +*_benchmark_app *_test_app /cmake-build-debug/ /mvnw diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java index 32571b4a1c6..4cb73cc3533 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/classtree/ClassFileTree.java @@ -4,12 +4,12 @@ package com.yahoo.abicheck.classtree; import java.io.IOException; import java.io.InputStream; import java.util.ArrayDeque; -import java.util.Arrays; import java.util.Collection; import java.util.Deque; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.jar.JarEntry; @@ -24,7 +24,7 @@ public abstract class ClassFileTree implements AutoCloseable { while (jarEntries.hasMoreElements()) { JarEntry entry = jarEntries.nextElement(); if (!entry.isDirectory() && entry.getName().endsWith(".class")) { - Deque<String> parts = new ArrayDeque<>(Arrays.asList(entry.getName().split("/"))); + Deque<String> parts = new ArrayDeque<>(List.of(entry.getName().split("/"))); String className = parts.removeLast(); Package pkg = rootPackages .computeIfAbsent(parts.removeFirst(), name -> new Package(null, name)); diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java index d592a2ce1e3..1631eac866f 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/AccessConversionTest.java @@ -5,17 +5,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import com.yahoo.abicheck.collector.Util; -import java.util.Arrays; import org.junit.jupiter.api.Test; import org.objectweb.asm.Opcodes; +import java.util.List; + public class AccessConversionTest { @Test public void testClassFlags() { // ACC_SUPER should be ignored assertEquals( - Arrays.asList("public", "abstract"), + List.of("public", "abstract"), Util.convertAccess( Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_ABSTRACT, Util.classFlags)); @@ -25,7 +26,7 @@ public class AccessConversionTest { public void testMethodFlags() { // ACC_DEPRECATED should be ignored assertEquals( - Arrays.asList("protected", "varargs"), + List.of("protected", "varargs"), Util.convertAccess( Opcodes.ACC_PROTECTED | Opcodes.ACC_VARARGS | Opcodes.ACC_DEPRECATED, Util.methodFlags)); @@ -34,7 +35,7 @@ public class AccessConversionTest { @Test public void testFieldFlags() { assertEquals( - Arrays.asList("private", "volatile"), + List.of("private", "volatile"), Util.convertAccess( Opcodes.ACC_PRIVATE | Opcodes.ACC_VOLATILE, Util.fieldFlags)); diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java index 3f90ba01848..1667a6b87d0 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/ClassFileTreeTest.java @@ -12,8 +12,8 @@ import com.google.common.collect.Iterables; import com.yahoo.abicheck.classtree.ClassFileTree; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.jar.JarEntry; import java.util.jar.JarFile; import org.junit.jupiter.api.Test; @@ -27,7 +27,7 @@ public class ClassFileTreeTest { JarEntry dirJarEntry = new JarEntry("com/yahoo/"); InputStream jarEntryStream = mock(InputStream.class); when(jarFile.entries()) - .thenReturn(Collections.enumeration(Arrays.asList(dirJarEntry, classJarEntry))); + .thenReturn(Collections.enumeration(List.of(dirJarEntry, classJarEntry))); when(jarFile.getInputStream(classJarEntry)).thenReturn(jarEntryStream); try (ClassFileTree cft = ClassFileTree.fromJar(jarFile)) { diff --git a/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java b/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java index 6821a03818e..9bc330ad130 100644 --- a/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java +++ b/abi-check-plugin/src/test/java/com/yahoo/abicheck/mojo/AbiCheckTest.java @@ -12,9 +12,9 @@ import com.yahoo.abicheck.Public; import com.yahoo.abicheck.classtree.ClassFileTree; import com.yahoo.abicheck.signature.JavaClassSignature; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.maven.plugin.logging.Log; import org.junit.jupiter.api.Test; import root.Root; @@ -30,9 +30,9 @@ public class AbiCheckTest { ClassFileTree.ClassFile rootPkgClass = mock(ClassFileTree.ClassFile.class); ClassFileTree.ClassFile subPkgClass = mock(ClassFileTree.ClassFile.class); - when(rootPkg.getSubPackages()).thenReturn(Collections.singleton(subPkg)); - when(rootPkg.getClassFiles()).thenReturn(Arrays.asList(rootPkgClass, rootPkgInfoClass)); - when(subPkg.getClassFiles()).thenReturn(Collections.singleton(subPkgClass)); + when(rootPkg.getSubPackages()).thenReturn(Set.of(subPkg)); + when(rootPkg.getClassFiles()).thenReturn(List.of(rootPkgClass, rootPkgInfoClass)); + when(subPkg.getClassFiles()).thenReturn(Set.of(subPkgClass)); when(rootPkgInfoClass.getName()).thenReturn("package-info.class"); when(rootPkgInfoClass.getInputStream()) @@ -68,16 +68,16 @@ public class AbiCheckTest { JavaClassSignature signatureA = new JavaClassSignature( "java.lang.Object", - Collections.emptySet(), - Collections.singletonList("public"), - Collections.singleton("public void foo()"), - Collections.singleton("public int bar")); + Set.of(), + List.of("public"), + Set.of("public void foo()"), + Set.of("public int bar")); JavaClassSignature signatureB = new JavaClassSignature( "java.lang.Exception", - Collections.singleton("java.lang.Runnable"), - Collections.singletonList("protected"), - Collections.singleton("public void foo(int)"), - Collections.singleton("public boolean bar")); + Set.of("java.lang.Runnable"), + List.of("protected"), + Set.of("public void foo(int)"), + Set.of("public boolean bar")); Map<String, JavaClassSignature> expected = ImmutableMap.<String, JavaClassSignature>builder() .put("test.Missing", signatureA) diff --git a/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java b/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java index 9514749138c..4079fc377d6 100644 --- a/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java +++ b/application/src/test/java/com/yahoo/application/container/handler/HeadersTestCase.java @@ -3,11 +3,7 @@ package com.yahoo.application.container.handler; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,9 +62,9 @@ public class HeadersTestCase { @Test void requireThatContainsValueWorksAsExpected() { Headers headers = new Headers(); - assertFalse(headers.containsValue(Arrays.asList("bar"))); + assertFalse(headers.containsValue(List.of("bar"))); headers.add("foo", "bar"); - assertTrue(headers.containsValue(Arrays.asList("bar"))); + assertTrue(headers.containsValue(List.of("bar"))); } @Test @@ -104,19 +100,19 @@ public class HeadersTestCase { Headers headers = new Headers(); assertNull(headers.get("foo")); headers.add("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); headers.add("foo", "baz"); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); } @Test void requireThatAddListWorksAsExpected() { Headers headers = new Headers(); assertNull(headers.get("foo")); - headers.add("foo", Arrays.asList("bar")); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - headers.add("foo", Arrays.asList("baz", "cox")); - assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); + headers.add("foo", List.of("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + headers.add("foo", List.of("baz", "cox")); + assertEquals(List.of("bar", "baz", "cox"), headers.get("foo")); } @Test @@ -124,16 +120,14 @@ public class HeadersTestCase { Headers headers = new Headers(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); - Map<String, List<String>> map = new HashMap<>(); - map.put("foo", Arrays.asList("baz", "cox")); - map.put("bar", Arrays.asList("cox")); + Map<String, List<String>> map = Map.of("foo", List.of("baz", "cox"), "bar", List.of("cox")); headers.addAll(map); - assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); - assertEquals(Arrays.asList("baz", "cox"), headers.get("bar")); + assertEquals(List.of("bar", "baz", "cox"), headers.get("foo")); + assertEquals(List.of("baz", "cox"), headers.get("bar")); } @Test @@ -141,19 +135,19 @@ public class HeadersTestCase { Headers headers = new Headers(); assertNull(headers.get("foo")); headers.put("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); headers.put("foo", "baz"); - assertEquals(Arrays.asList("baz"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("foo")); } @Test void requireThatPutListWorksAsExpected() { Headers headers = new Headers(); assertNull(headers.get("foo")); - headers.put("foo", Arrays.asList("bar")); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - headers.put("foo", Arrays.asList("baz", "cox")); - assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); + headers.put("foo", List.of("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + headers.put("foo", List.of("baz", "cox")); + assertEquals(List.of("baz", "cox"), headers.get("foo")); } @Test @@ -161,24 +155,22 @@ public class HeadersTestCase { Headers headers = new Headers(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); - Map<String, List<String>> map = new HashMap<>(); - map.put("foo", Arrays.asList("baz", "cox")); - map.put("bar", Arrays.asList("cox")); + Map<String, List<String>> map = Map.of("foo", List.of("baz", "cox"), "bar", List.of("cox")); headers.putAll(map); - assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); - assertEquals(Arrays.asList("cox"), headers.get("bar")); + assertEquals(List.of("baz", "cox"), headers.get("foo")); + assertEquals(List.of("cox"), headers.get("bar")); } @Test void requireThatRemoveWorksAsExpected() { Headers headers = new Headers(); - headers.put("foo", Arrays.asList("bar", "baz")); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); - assertEquals(Arrays.asList("bar", "baz"), headers.remove("foo")); + headers.put("foo", List.of("bar", "baz")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); + assertEquals(List.of("bar", "baz"), headers.remove("foo")); assertNull(headers.get("foo")); assertNull(headers.remove("foo")); } @@ -186,11 +178,11 @@ public class HeadersTestCase { @Test void requireThatRemoveStringWorksAsExpected() { Headers headers = new Headers(); - headers.put("foo", Arrays.asList("bar", "baz")); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + headers.put("foo", List.of("bar", "baz")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); assertTrue(headers.remove("foo", "bar")); assertFalse(headers.remove("foo", "cox")); - assertEquals(Arrays.asList("baz"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("foo")); assertTrue(headers.remove("foo", "baz")); assertFalse(headers.remove("foo", "cox")); assertNull(headers.get("foo")); @@ -201,8 +193,8 @@ public class HeadersTestCase { Headers headers = new Headers(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); headers.clear(); assertNull(headers.get("foo")); assertNull(headers.get("bar")); @@ -213,14 +205,14 @@ public class HeadersTestCase { Headers headers = new Headers(); assertNull(headers.get("foo")); headers.add("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); } @Test void requireThatGetFirstWorksAsExpected() { Headers headers = new Headers(); assertNull(headers.getFirst("foo")); - headers.add("foo", Arrays.asList("bar", "baz")); + headers.add("foo", List.of("bar", "baz")); assertEquals("bar", headers.getFirst("foo")); } @@ -228,28 +220,28 @@ public class HeadersTestCase { void requireThatIsTrueWorksAsExpected() { Headers headers = new Headers(); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true")); + headers.put("foo", List.of("true")); assertTrue(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true", "true")); + headers.put("foo", List.of("true", "true")); assertTrue(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true", "false")); + headers.put("foo", List.of("true", "false")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false", "true")); + headers.put("foo", List.of("false", "true")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false", "false")); + headers.put("foo", List.of("false", "false")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false")); + headers.put("foo", List.of("false")); assertFalse(headers.isTrue("foo")); } @Test void requireThatKeySetWorksAsExpected() { Headers headers = new Headers(); - assertEquals(Collections.<Set<String>>emptySet(), headers.keySet()); + assertEquals(Set.of(), headers.keySet()); headers.add("foo", "bar"); - assertEquals(new HashSet<>(Arrays.asList("foo")), headers.keySet()); + assertEquals(Set.of("foo"), headers.keySet()); headers.add("bar", "baz"); - assertEquals(new HashSet<>(Arrays.asList("foo", "bar")), headers.keySet()); + assertEquals(Set.of("foo", "bar"), headers.keySet()); } @Test @@ -259,34 +251,34 @@ public class HeadersTestCase { headers.add("foo", "bar"); Collection<List<String>> values = headers.values(); assertEquals(1, values.size()); - assertTrue(values.contains(Arrays.asList("bar"))); + assertTrue(values.contains(List.of("bar"))); headers.add("bar", "baz"); values = headers.values(); assertEquals(2, values.size()); - assertTrue(values.contains(Arrays.asList("bar"))); - assertTrue(values.contains(Arrays.asList("baz"))); + assertTrue(values.contains(List.of("bar"))); + assertTrue(values.contains(List.of("baz"))); } @Test void requireThatEntrySetWorksAsExpected() { Headers headers = new Headers(); - assertEquals(Collections.emptySet(), headers.entrySet()); - headers.put("foo", Arrays.asList("bar", "baz")); + assertEquals(Set.of(), headers.entrySet()); + headers.put("foo", List.of("bar", "baz")); Set<Map.Entry<String, List<String>>> entries = headers.entrySet(); assertEquals(1, entries.size()); Map.Entry<String, List<String>> entry = entries.iterator().next(); assertNotNull(entry); assertEquals("foo", entry.getKey()); - assertEquals(Arrays.asList("bar", "baz"), entry.getValue()); + assertEquals(List.of("bar", "baz"), entry.getValue()); } @Test void requireThatEntriesWorksAsExpected() { Headers headers = new Headers(); - assertEquals(Collections.emptyList(), headers.entries()); - headers.put("foo", Arrays.asList("bar", "baz")); + assertEquals(List.of(), headers.entries()); + headers.put("foo", List.of("bar", "baz")); List<Map.Entry<String, String>> entries = headers.entries(); assertEquals(2, entries.size()); diff --git a/build_settings.cmake b/build_settings.cmake index 3477531cb56..1549ac83c74 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -90,12 +90,6 @@ else() message("-- liburing not found") endif() -if(VESPA_OS_DISTRO_COMBINED STREQUAL "debian 10") - unset(VESPA_XXHASH_DEFINE) -else() - set(VESPA_XXHASH_DEFINE "-DXXH_INLINE_ALL") -endif() - # Disable dangling reference and overloaded virtual warnings when using gcc 13 # Disable stringop-oveflow, stringop-overread and array-bounds warning when using gcc 13. # The latter heuristics are sufficiently broken to be useless in practice. @@ -111,7 +105,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND VESPA_USE_LTO) endif() # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DXXH_INLINE_ALL -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") # AddressSanitizer/ThreadSanitizer work for both GCC and Clang if (VESPA_USE_SANITIZER) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}") diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java index cbe5e97374c..e8c0e7a4800 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java @@ -7,9 +7,7 @@ import com.yahoo.container.plugin.util.JarFiles; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.jar.Manifest; @@ -37,7 +35,7 @@ public class AnalyzeBundle { static List<Export> exportedPackages(File jarFile) { var manifest = getOsgiManifest(jarFile); - if (manifest == null) return Collections.emptyList(); + if (manifest == null) return List.of(); try { return parseExports(manifest); } catch (Exception e) { @@ -55,9 +53,9 @@ public class AnalyzeBundle { private static List<String> nonPublicApiPackages(File jarFile) { var manifest = getOsgiManifest(jarFile); - if (manifest == null) return Collections.emptyList(); + if (manifest == null) return List.of(); return getMainAttributeValue(manifest, "X-JDisc-Non-PublicApi-Export-Package") - .map(s -> Arrays.asList(s.split(","))) + .map(s -> List.of(s.split(","))) .orElseGet(ArrayList::new); } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index 6b8f3f6ba12..5546d3c4f32 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -14,8 +14,8 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -55,9 +55,9 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { Analyze.getClassName(Type.getReturnType(desc)).ifPresent(imports::add); - Arrays.asList(Type.getArgumentTypes(desc)).forEach(argType -> Analyze.getClassName(argType).ifPresent(imports::add)); + List.of(Type.getArgumentTypes(desc)).forEach(argType -> Analyze.getClassName(argType).ifPresent(imports::add)); if (exceptions != null) { - Arrays.asList(exceptions).forEach(ex -> Analyze.internalNameToClassName(ex).ifPresent(imports::add)); + List.of(exceptions).forEach(ex -> Analyze.internalNameToClassName(ex).ifPresent(imports::add)); } AnalyzeSignatureVisitor.analyzeMethod(signature, this); @@ -83,7 +83,7 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { } addImportWithInternalName(superName); - Arrays.asList(interfaces).forEach(this::addImportWithInternalName); + List.of(interfaces).forEach(this::addImportWithInternalName); AnalyzeSignatureVisitor.analyzeClass(signature, this); } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java index 788fa346a37..d7991810024 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -9,8 +9,8 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -70,7 +70,7 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { addImportWithInternalName(owner); - Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); + List.of(Type.getArgumentTypes(desc)).forEach(this::addImport); addImport(Type.getReturnType(desc)); } @@ -112,7 +112,7 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { addImport((Type) arg); } else if (arg instanceof Handle) { addImportWithInternalName(((Handle) arg).getOwner()); - Arrays.asList(Type.getArgumentTypes(desc)).forEach(this::addImport); + List.of(Type.getArgumentTypes(desc)).forEach(this::addImport); } else if ( ! (arg instanceof Number) && ! (arg instanceof String)) { throw new AssertionError("Unexpected type " + arg.getClass() + " with value '" + arg + "'"); } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateSourcesMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateSourcesMojo.java index adcbdb68553..74acf3dffa1 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateSourcesMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateSourcesMojo.java @@ -16,9 +16,9 @@ import org.codehaus.plexus.component.annotations.Requirement; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Properties; import java.util.regex.Pattern; @@ -145,7 +145,7 @@ public class GenerateSourcesMojo extends AbstractMojo { if (parts.length <= 3) { return mavenVersion; } else { - return stringJoin(Arrays.asList(parts).subList(0, 3), "."); + return stringJoin(List.of(parts).subList(0, 3), "."); } } } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java index af403946f39..c66aa61a8ee 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/util/JarFiles.java @@ -4,7 +4,6 @@ package com.yahoo.container.plugin.util; import java.io.File; import java.io.InputStream; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.jar.JarFile; @@ -25,8 +24,8 @@ public class JarFiles { .map(s -> Arrays.stream(s.split(",")) .map(ArtifactId::fromStringValue) .toList()) - .orElse(Collections.emptyList())) - .orElse(Collections.emptyList()); + .orElse(List.of())) + .orElse(List.of()); } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java index caaa8d5c6c0..7ad1afff01d 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java @@ -6,7 +6,6 @@ import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; import com.yahoo.container.plugin.osgi.ImportPackages.Import; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -23,8 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ public class ImportPackageTest { private static final String PACKAGE_NAME = "com.yahoo.exported"; - private final Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); - private final Map<String, Export> exports = exportByPackageName(new Export(List.of(PACKAGE_NAME), Collections.emptyList())); + private final Set<String> referencedPackages = Set.of(PACKAGE_NAME); + private final Map<String, Export> exports = exportByPackageName(new Export(List.of(PACKAGE_NAME), List.of())); private final Map<String, Export> exportsWithVersion = exportByPackageName( new Export(List.of(PACKAGE_NAME), List.of(new Parameter("version", "1.3")))); diff --git a/client/go/go.mod b/client/go/go.mod index 9fe7096fa36..ba0af5a763e 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -8,7 +8,7 @@ require ( github.com/fatih/color v1.16.0 // This is the most recent version compatible with Go 1.20. Upgrade when we upgrade our Go version github.com/go-json-experiment/json v0.0.0-20230324203220-04923b7a9528 - github.com/klauspost/compress v1.17.7 + github.com/klauspost/compress v1.17.8 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c diff --git a/client/go/go.sum b/client/go/go.sum index be977057cfd..d985c9e7ffc 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -26,6 +26,8 @@ github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2e github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/client/go/internal/admin/vespa-wrapper/standalone/start.go b/client/go/internal/admin/vespa-wrapper/standalone/start.go index a3703ce930c..16e76562b99 100644 --- a/client/go/internal/admin/vespa-wrapper/standalone/start.go +++ b/client/go/internal/admin/vespa-wrapper/standalone/start.go @@ -41,6 +41,7 @@ func StartStandaloneContainer(extraArgs []string) int { c := jvm.NewStandaloneContainer(serviceName) jvmOpts := c.JvmOptions() jvmOpts.AddOption("-DOnnxBundleActivator.skip=true") + jvmOpts.AddOption("-DLlamaBundleActivator.skip=true") for _, extra := range extraArgs { jvmOpts.AddOption(extra) } diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 789267f9461..7074aa923ec 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -789,10 +789,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@9.0.0": - version "9.0.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.0.0.tgz#1a9e4b4c96d8c7886e0110ed310a0135144a1691" - integrity sha512-RThY/MnKrhubF6+s1JflwUjPEsnCEmYCWwqa/aRISKWNXGZ9epUwft4bUMM35SdKF9xvBrLydAM1RDHd1Z//ZQ== +"@eslint/js@9.1.1": + version "9.1.1" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.1.1.tgz#eb0f82461d12779bbafc1b5045cde3143d350a8a" + integrity sha512-5WoDz3Y19Bg2BnErkZTp0en+c/i9PvgFS7MBe1+m60HjFr0hrphlAGp4yzI7pxpt4xShln4ZyYp4neJm8hmOkQ== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -863,10 +863,10 @@ dependencies: prop-types "^15.8.1" -"@humanwhocodes/config-array@^0.12.3": - version "0.12.3" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.12.3.tgz#a6216d90f81a30bedd1d4b5d799b47241f318072" - integrity sha512-jsNnTBlMWuTpDkeE3on7+dWJi0D6fdDfeANj/w7MpS8ztROCoLvIO2nG0CcFj+E4k8j4QrSTh4Oryi3i2G669g== +"@humanwhocodes/config-array@^0.13.0": + version "0.13.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.13.0.tgz#fb907624df3256d04b9aa2df50d7aa97ec648748" + integrity sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw== dependencies: "@humanwhocodes/object-schema" "^2.0.3" debug "^4.3.1" @@ -882,6 +882,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3" integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== +"@humanwhocodes/retry@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.2.3.tgz#c9aa036d1afa643f1250e83150f39efb3a15a631" + integrity sha512-X38nUbachlb01YMlvPFojKoiXq+LzZvuSce70KPMPdeM1Rj03k4dR7lDslhbqXn3Ang4EU3+EAmwEAsbrjHW3g== + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -1315,85 +1320,90 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.15.3": - version "1.15.3" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.15.3.tgz#d2509048d69dbb72d5389a14945339f1430b2d3c" - integrity sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w== - -"@rollup/rollup-android-arm-eabi@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.14.0.tgz#57936f50d0335e2e7bfac496d209606fa516add4" - integrity sha512-jwXtxYbRt1V+CdQSy6Z+uZti7JF5irRKF8hlKfEnF/xJpcNGuuiZMBvuoYM+x9sr9iWGnzrlM0+9hvQ1kgkf1w== - -"@rollup/rollup-android-arm64@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.14.0.tgz#81bba83b37382a2d0e30ceced06c8d3d85138054" - integrity sha512-fI9nduZhCccjzlsA/OuAwtFGWocxA4gqXGTLvOyiF8d+8o0fZUeSztixkYjcGq1fGZY3Tkq4yRvHPFxU+jdZ9Q== - -"@rollup/rollup-darwin-arm64@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.14.0.tgz#a371bd723a5c4c4a33376da72abfc3938066842b" - integrity sha512-BcnSPRM76/cD2gQC+rQNGBN6GStBs2pl/FpweW8JYuz5J/IEa0Fr4AtrPv766DB/6b2MZ/AfSIOSGw3nEIP8SA== - -"@rollup/rollup-darwin-x64@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.14.0.tgz#8baf2fda277c9729125017c65651296282412886" - integrity sha512-LDyFB9GRolGN7XI6955aFeI3wCdCUszFWumWU0deHA8VpR3nWRrjG6GtGjBrQxQKFevnUTHKCfPR4IvrW3kCgQ== - -"@rollup/rollup-linux-arm-gnueabihf@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.14.0.tgz#822830a8f7388d5b81d04c69415408d3bab1079b" - integrity sha512-ygrGVhQP47mRh0AAD0zl6QqCbNsf0eTo+vgwkY6LunBcg0f2Jv365GXlDUECIyoXp1kKwL5WW6rsO429DBY/bA== - -"@rollup/rollup-linux-arm64-gnu@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.14.0.tgz#e20fbe1bd4414c7119f9e0bba8ad17a6666c8365" - integrity sha512-x+uJ6MAYRlHGe9wi4HQjxpaKHPM3d3JjqqCkeC5gpnnI6OWovLdXTpfa8trjxPLnWKyBsSi5kne+146GAxFt4A== - -"@rollup/rollup-linux-arm64-musl@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.14.0.tgz#13f475596a62e1924f13fe1c8cf2c40e09a99b47" - integrity sha512-nrRw8ZTQKg6+Lttwqo6a2VxR9tOroa2m91XbdQ2sUUzHoedXlsyvY1fN4xWdqz8PKmf4orDwejxXHjh7YBGUCA== - -"@rollup/rollup-linux-powerpc64le-gnu@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.14.0.tgz#6a431c441420d1c510a205e08c6673355a0a2ea9" - integrity sha512-xV0d5jDb4aFu84XKr+lcUJ9y3qpIWhttO3Qev97z8DKLXR62LC3cXT/bMZXrjLF9X+P5oSmJTzAhqwUbY96PnA== - -"@rollup/rollup-linux-riscv64-gnu@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.14.0.tgz#53d9448962c3f9ed7a1672269655476ea2d67567" - integrity sha512-SDDhBQwZX6LPRoPYjAZWyL27LbcBo7WdBFWJi5PI9RPCzU8ijzkQn7tt8NXiXRiFMJCVpkuMkBf4OxSxVMizAw== - -"@rollup/rollup-linux-s390x-gnu@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.14.0.tgz#95f0c133b324da3e7e5c7d12855e0eb71d21a946" - integrity sha512-RxB/qez8zIDshNJDufYlTT0ZTVut5eCpAZ3bdXDU9yTxBzui3KhbGjROK2OYTTor7alM7XBhssgoO3CZ0XD3qA== - -"@rollup/rollup-linux-x64-gnu@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.14.0.tgz#820ada75c68ead1acc486e41238ca0d8f8531478" - integrity sha512-C6y6z2eCNCfhZxT9u+jAM2Fup89ZjiG5pIzZIDycs1IwESviLxwkQcFRGLjnDrP+PT+v5i4YFvlcfAs+LnreXg== - -"@rollup/rollup-linux-x64-musl@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.14.0.tgz#ca74f22e125efbe94c1148d989ef93329b464443" - integrity sha512-i0QwbHYfnOMYsBEyjxcwGu5SMIi9sImDVjDg087hpzXqhBSosxkE7gyIYFHgfFl4mr7RrXksIBZ4DoLoP4FhJg== - -"@rollup/rollup-win32-arm64-msvc@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.14.0.tgz#269023332297051d037a9593dcba92c10fef726b" - integrity sha512-Fq52EYb0riNHLBTAcL0cun+rRwyZ10S9vKzhGKKgeD+XbwunszSY0rVMco5KbOsTlwovP2rTOkiII/fQ4ih/zQ== - -"@rollup/rollup-win32-ia32-msvc@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.14.0.tgz#d7701438daf964011fd7ca33e3f13f3ff5129e7b" - integrity sha512-e/PBHxPdJ00O9p5Ui43+vixSgVf4NlLsmV6QneGERJ3lnjIua/kim6PRFe3iDueT1rQcgSkYP8ZBBXa/h4iPvw== - -"@rollup/rollup-win32-x64-msvc@4.14.0": - version "4.14.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.14.0.tgz#0bb7ac3cd1c3292db1f39afdabfd03ccea3a3d34" - integrity sha512-aGg7iToJjdklmxlUlJh/PaPNa4PmqHfyRMLunbL3eaMO0gp656+q1zOKkpJ/CVe9CryJv6tAN1HDoR8cNGzkag== +"@remix-run/router@1.16.0": + version "1.16.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.16.0.tgz#0e10181e5fec1434eb071a9bc4bdaac843f16dcc" + integrity sha512-Quz1KOffeEf/zwkCBM3kBtH4ZoZ+pT3xIXBG4PPW/XFtDP7EGhtTiC2+gpL9GnR7+Qdet5Oa6cYSvwKYg6kN9Q== + +"@rollup/rollup-android-arm-eabi@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.15.0.tgz#28c9c79c5baccb59a96afcf60e428ea6965a5579" + integrity sha512-O63bJ7p909pRRQfOJ0k/Jp8gNFMud+ZzLLG5EBWquylHxmRT2k18M2ifg8WyjCgFVdpA7+rI0YZ8EkAtg6dSUw== + +"@rollup/rollup-android-arm64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.15.0.tgz#a2bdafdb753ece571956289a5ba8c37af748bd0c" + integrity sha512-5UywPdmC9jiVOShjQx4uuIcnTQOf85iA4jgg8bkFoH5NYWFfAfrJpv5eeokmTdSmYwUTT5IrcrBCJNkowhrZDA== + +"@rollup/rollup-darwin-arm64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.15.0.tgz#4cb44cfec3068f6f76f70463ccc25f3e245af06a" + integrity sha512-hNkt75uFfWpRxHItCBmbS0ba70WnibJh6yz60WShSWITLlVRbkvAu1E/c7RlliPY4ajhqJd0UPZz//gNalTd4g== + +"@rollup/rollup-darwin-x64@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.15.0.tgz#1035bfbf53e6acf16771191f41c3d3aff089e8f1" + integrity sha512-HnC5bTP7qdfO9nUw/mBhNcjOEZfbS8NwV+nFegiMhYOn1ATAGZF4kfAxR9BuZevBrebWCxMmxm8NCU1CUoz+wQ== + +"@rollup/rollup-linux-arm-gnueabihf@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.15.0.tgz#0036b835f17ca9e84c188c419399493bd5739986" + integrity sha512-QGOIQIJZeIIqMsc4BUGe8TnV4dkXhSW2EhaQ1G4LqMUNpkyeLztvlDlOoNHn7SR7a4dBANdcEbPkkEzz3rzjzA== + +"@rollup/rollup-linux-arm-musleabihf@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.15.0.tgz#c44420167203400ba7a707f8205413c2817cdaeb" + integrity sha512-PS/Cp8CinYgoysQ8i4UXYH/TZl06fXszvY/RDkyBYgUB1+tKyOMS925/4FZhfrhkl3XQEKjMc3BKtsxpB9Tz9Q== + +"@rollup/rollup-linux-arm64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.15.0.tgz#531d3792e1526583ecd794ceee0ab980d79813dd" + integrity sha512-XzOsnD6lGDP+k+vGgTYAryVGu8N89qpjMN5BVFUj75dGVFP3FzIVAufJAraxirpDwEQZA7Gjs0Vo5p4UmnnjsA== + +"@rollup/rollup-linux-arm64-musl@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.15.0.tgz#86376eaa6d65a860a046e0dfe285a51792bc2026" + integrity sha512-+ScJA4Epbx/ZQGjDnbvTAcb8ZD06b+TlIka2UkujbKf1I/A+yrvEcJwG3/27zMmvcWMQyeCJhbL9TlSjzL0B7Q== + +"@rollup/rollup-linux-powerpc64le-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.15.0.tgz#6adf69ce27d1266dbb86eeac237ad5dd4d4a1f28" + integrity sha512-1cUSvYgnyTakM4FDyf/GxUCDcqmj/hUh1NOizEOJU7+D5xEfFGCxgcNOs3hYBeRMUCcGmGkt01EhD3ILgKpGHQ== + +"@rollup/rollup-linux-riscv64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.15.0.tgz#961c290372d170f588ebf65c145c485c4aad7005" + integrity sha512-3A1FbHDbBUvpJXFAZwVsiROIcstVHP9AX/cwnyIhAp+xyQ1cBCxywKtuzmw0Av1MDNNg/y/9dDHtNypfRa8bdw== + +"@rollup/rollup-linux-s390x-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.15.0.tgz#cb43301e10f17f0a642416c5d3d82a26cf430fa8" + integrity sha512-hYPbhg9ow6/mXIkojc8LOeiip2sCTuw1taWyoOXTOWk9vawIXz8x7B4KkgWUAtvAElssxhSyEXr2EZycH/FGzQ== + +"@rollup/rollup-linux-x64-gnu@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.15.0.tgz#25c1fb87b2255949ee7ff6956e205710c5d7c414" + integrity sha512-511qln5mPSUKwv7HI28S1jCD1FK+2WbX5THM9A9annr3c1kzmfnf8Oe3ZakubEjob3IV6OPnNNcesfy+adIrmw== + +"@rollup/rollup-linux-x64-musl@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.15.0.tgz#145745e339e282c7afc36142bd5a3f9495c7c681" + integrity sha512-4qKKGTDIv2bQZ+afhPWqPL+94+dLtk4lw1iwbcylKlLNqQ/Yyjof2CFYBxf6npiDzPV+zf4EWRiHb26/4Vsm9w== + +"@rollup/rollup-win32-arm64-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.15.0.tgz#95dae687b645a25aab3a082d987556f58274ffbe" + integrity sha512-nEtaFBHp1OnbOf+tz66DtID579sNRHGgMC23to8HUyVuOCpCMD0CvRNqiDGLErLNnwApWIUtUl1VvuovCWUxwg== + +"@rollup/rollup-win32-ia32-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.15.0.tgz#acbd48f10093e6cd52f99ad004966433a49cb362" + integrity sha512-5O49NykwSgX6iT2HgZ6cAoGHt6T/FqNMB5OqFOGxU/y1GyFSHquox1sK2OqApQc0ANxiHFQEMNDLNVCL7AUDnQ== + +"@rollup/rollup-win32-x64-msvc@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.15.0.tgz#68bb584231dfc8e36bb7ad5317dfd1fd2563a6f7" + integrity sha512-YA0hTwCunmKNeTOFWdJuKhdXse9jBqgo34FDo+9aS0spfCkp+wj0o1bCcOOTu+0P48O95GTfkLTAaVonwNuIdQ== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -2659,16 +2669,17 @@ eslint-visitor-keys@^4.0.0: integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== eslint@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.0.0.tgz#6270548758e390343f78c8afd030566d86927d40" - integrity sha512-IMryZ5SudxzQvuod6rUdIUz29qFItWx281VhtFVc2Psy/ZhlCeD/5DT6lBIJ4H3G+iamGJoTln1v+QSuPw0p7Q== + version "9.1.1" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.1.1.tgz#39ec657ccd12813cb4a1dab2f9229dcc6e468271" + integrity sha512-b4cRQ0BeZcSEzPpY2PjFY70VbO32K7BStTGtBsnIGdTSEEQzBi8hPBcGQmTG2zUvFr9uLe0TK42bw8YszuHEqg== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^3.0.2" - "@eslint/js" "9.0.0" - "@humanwhocodes/config-array" "^0.12.3" + "@eslint/js" "9.1.1" + "@humanwhocodes/config-array" "^0.13.0" "@humanwhocodes/module-importer" "^1.0.1" + "@humanwhocodes/retry" "^0.2.3" "@nodelib/fs.walk" "^1.2.8" ajv "^6.12.4" chalk "^4.0.0" @@ -2684,7 +2695,6 @@ eslint@^9.0.0: file-entry-cache "^8.0.0" find-up "^5.0.0" glob-parent "^6.0.2" - graphemer "^1.4.0" ignore "^5.2.0" imurmurhash "^0.1.4" is-glob "^4.0.0" @@ -3073,11 +3083,6 @@ graceful-fs@^4.2.4, graceful-fs@^4.2.9: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -graphemer@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" - integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== - has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -4718,19 +4723,19 @@ react-refresh@^0.14.0: integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== react-router-dom@^6: - version "6.22.3" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.22.3.tgz#9781415667fd1361a475146c5826d9f16752a691" - integrity sha512-7ZILI7HjcE+p31oQvwbokjk6OA/bnFxrhJ19n82Ex9Ph8fNAq+Hm/7KchpMGlTgWhUxRHMMCut+vEtNpWpowKw== + version "6.23.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.23.0.tgz#8b80ad92ad28f4dc38972e92d84b4c208150545a" + integrity sha512-Q9YaSYvubwgbal2c9DJKfx6hTNoBp3iJDsl+Duva/DwxoJH+OTXkxGpql4iUK2sla/8z4RpjAm6EWx1qUDuopQ== dependencies: - "@remix-run/router" "1.15.3" - react-router "6.22.3" + "@remix-run/router" "1.16.0" + react-router "6.23.0" -react-router@6.22.3: - version "6.22.3" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.22.3.tgz#9d9142f35e08be08c736a2082db5f0c9540a885e" - integrity sha512-dr2eb3Mj5zK2YISHK++foM9w4eBnO23eKnZEDs7c880P6oKbrjz/Svg9+nxqtHQK+oMW4OtjZca0RqPglXxguQ== +react-router@6.23.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.23.0.tgz#2f2d7492c66a6bdf760be4c6bdf9e1d672fa154b" + integrity sha512-wPMZ8S2TuPadH0sF5irFGjkNLIcRvOSaEe7v+JER8508dyJumm6XZB1u5kztlX0RVq6AzRVndzqcUh6sFIauzA== dependencies: - "@remix-run/router" "1.15.3" + "@remix-run/router" "1.16.0" react-textarea-autosize@8.3.4: version "8.3.4" @@ -4879,27 +4884,28 @@ reusify@^1.0.4: integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== rollup@^4.13.0: - version "4.14.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.14.0.tgz#c3e2cd479f1b2358b65c1f810fa05b51603d7be8" - integrity sha512-Qe7w62TyawbDzB4yt32R0+AbIo6m1/sqO7UPzFS8Z/ksL5mrfhA0v4CavfdmFav3D+ub4QeAgsGEe84DoWe/nQ== + version "4.15.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.15.0.tgz#3be428e4fe86297b1b3448f29515d978593d9d9a" + integrity sha512-i0ir57IMF5o7YvNYyUNeIGG+IZaaucnGZAOsSctO2tPLXlCEaZzyBa+QhpHNSgtpyLMoDev2DyN6a7J1dQA8Tw== dependencies: "@types/estree" "1.0.5" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.14.0" - "@rollup/rollup-android-arm64" "4.14.0" - "@rollup/rollup-darwin-arm64" "4.14.0" - "@rollup/rollup-darwin-x64" "4.14.0" - "@rollup/rollup-linux-arm-gnueabihf" "4.14.0" - "@rollup/rollup-linux-arm64-gnu" "4.14.0" - "@rollup/rollup-linux-arm64-musl" "4.14.0" - "@rollup/rollup-linux-powerpc64le-gnu" "4.14.0" - "@rollup/rollup-linux-riscv64-gnu" "4.14.0" - "@rollup/rollup-linux-s390x-gnu" "4.14.0" - "@rollup/rollup-linux-x64-gnu" "4.14.0" - "@rollup/rollup-linux-x64-musl" "4.14.0" - "@rollup/rollup-win32-arm64-msvc" "4.14.0" - "@rollup/rollup-win32-ia32-msvc" "4.14.0" - "@rollup/rollup-win32-x64-msvc" "4.14.0" + "@rollup/rollup-android-arm-eabi" "4.15.0" + "@rollup/rollup-android-arm64" "4.15.0" + "@rollup/rollup-darwin-arm64" "4.15.0" + "@rollup/rollup-darwin-x64" "4.15.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.15.0" + "@rollup/rollup-linux-arm-musleabihf" "4.15.0" + "@rollup/rollup-linux-arm64-gnu" "4.15.0" + "@rollup/rollup-linux-arm64-musl" "4.15.0" + "@rollup/rollup-linux-powerpc64le-gnu" "4.15.0" + "@rollup/rollup-linux-riscv64-gnu" "4.15.0" + "@rollup/rollup-linux-s390x-gnu" "4.15.0" + "@rollup/rollup-linux-x64-gnu" "4.15.0" + "@rollup/rollup-linux-x64-musl" "4.15.0" + "@rollup/rollup-win32-arm64-msvc" "4.15.0" + "@rollup/rollup-win32-ia32-msvc" "4.15.0" + "@rollup/rollup-win32-x64-msvc" "4.15.0" fsevents "~2.3.2" rsvp@^4.8.4: @@ -5515,9 +5521,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^5.0.5: - version "5.2.8" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.8.tgz#a99e09939f1a502992381395ce93efa40a2844aa" - integrity sha512-OyZR+c1CE8yeHw5V5t59aXsUPPVTHMDjEZz8MgguLL/Q7NblxhZUlTu9xSPqlsUO/y+X7dlU05jdhvyycD55DA== + version "5.2.10" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.10.tgz#2ac927c91e99d51b376a5c73c0e4b059705f5bd7" + integrity sha512-PAzgUZbP7msvQvqdSD+ErD5qGnSFiGOoWmV5yAKUEI0kdhjbH6nMWVyZQC/hSc4aXwc0oJ9aEdIiF9Oje0JFCw== dependencies: esbuild "^0.20.1" postcss "^8.4.38" diff --git a/client/src/main/java/ai/vespa/client/dsl/Sources.java b/client/src/main/java/ai/vespa/client/dsl/Sources.java index b6dd1b06536..cecf27aa240 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Sources.java +++ b/client/src/main/java/ai/vespa/client/dsl/Sources.java @@ -3,7 +3,6 @@ package ai.vespa.client.dsl; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -19,7 +18,7 @@ public class Sources { } Sources(Select select, String searchDefinition) { - this(select, Collections.singletonList(searchDefinition)); + this(select, List.of(searchDefinition)); } Sources(Select select, String searchDefinition, String... others) { diff --git a/client/src/test/java/ai/vespa/client/dsl/QTest.java b/client/src/test/java/ai/vespa/client/dsl/QTest.java index 3dbf714ed62..75e74c8919a 100644 --- a/client/src/test/java/ai/vespa/client/dsl/QTest.java +++ b/client/src/test/java/ai/vespa/client/dsl/QTest.java @@ -3,10 +3,9 @@ package ai.vespa.client.dsl; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -224,9 +223,9 @@ class QTest { String q = Q.select("*") .from("sd1") .where(Q.wand("f1", stringIntMap("a", 1, "b", 2, "c", 3))) - .and(Q.wand("f2", Arrays.asList(Arrays.asList(1, 1), Arrays.asList(2, 2)))) + .and(Q.wand("f2", List.of(List.of(1, 1), List.of(2, 2)))) .and( - Q.wand("f3", Arrays.asList(Arrays.asList(1, 1), Arrays.asList(2, 2))) + Q.wand("f3", List.of(List.of(1, 1), List.of(2, 2))) .annotate(A.a("scoreThreshold", 0.13)) ) .build(); @@ -421,7 +420,7 @@ class QTest { { String q1 = Q.p("f1").containsPhrase("p1", "p2", "p3") .build(); - String q2 = Q.p("f1").containsPhrase(Arrays.asList("p1", "p2", "p3")) + String q2 = Q.p("f1").containsPhrase(List.of("p1", "p2", "p3")) .build(); assertEquals(q1, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\")"); assertEquals(q2, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\")"); @@ -429,7 +428,7 @@ class QTest { { String q1 = Q.p("f1").containsNear("p1", "p2", "p3") .build(); - String q2 = Q.p("f1").containsNear(Arrays.asList("p1", "p2", "p3")) + String q2 = Q.p("f1").containsNear(List.of("p1", "p2", "p3")) .build(); assertEquals(q1, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\")"); assertEquals(q2, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\")"); @@ -437,7 +436,7 @@ class QTest { { String q1 = Q.p("f1").containsOnear("p1", "p2", "p3") .build(); - String q2 = Q.p("f1").containsOnear(Arrays.asList("p1", "p2", "p3")) + String q2 = Q.p("f1").containsOnear(List.of("p1", "p2", "p3")) .build(); assertEquals(q1, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\")"); assertEquals(q2, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\")"); @@ -445,7 +444,7 @@ class QTest { { String q1 = Q.p("f1").containsEquiv("p1", "p2", "p3") .build(); - String q2 = Q.p("f1").containsEquiv(Arrays.asList("p1", "p2", "p3")) + String q2 = Q.p("f1").containsEquiv(List.of("p1", "p2", "p3")) .build(); assertEquals(q1, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\")"); assertEquals(q2, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\")"); @@ -498,7 +497,7 @@ class QTest { @Test void use_contains_instead_of_contains_equiv_when_input_size_is_1() { - String q = Q.p("f1").containsEquiv(Collections.singletonList("p1")) + String q = Q.p("f1").containsEquiv(List.of("p1")) .build(); assertEquals(q, "yql=select * from sources * where f1 contains \"p1\""); @@ -506,16 +505,16 @@ class QTest { @Test void contains_phrase_near_onear_equiv_empty_list_should_throw_illegal_argument_exception() { - assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsPhrase(Collections.emptyList()) + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsPhrase(List.of()) .build()); - assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsNear(Collections.emptyList()) + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsNear(List.of()) .build()); - assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsOnear(Collections.emptyList()) + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsOnear(List.of()) .build()); - assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsEquiv(Collections.emptyList()) + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsEquiv(List.of()) .build()); } @@ -586,7 +585,7 @@ class QTest { @Test void arbitrary_annotations() { - Annotation a = A.a("a1", "v1", "a2", 2, "a3", stringObjMap("k", "v", "k2", 1), "a4", 4D, "a5", Arrays.asList(1, 2, 3)); + Annotation a = A.a("a1", "v1", "a2", 2, "a3", stringObjMap("k", "v", "k2", 1), "a4", 4D, "a5", List.of(1, 2, 3)); assertEquals(a.toString(), "{\"a1\":\"v1\",\"a2\":2,\"a3\":{\"k\":\"v\",\"k2\":1},\"a4\":4.0,\"a5\":[1,2,3]}"); } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index eff4e4125e9..98bef7df402 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -141,6 +141,7 @@ <include>com.microsoft.onnxruntime:onnxruntime:jar:${onnxruntime.vespa.version}:test</include> <include>com.thaiopensource:jing:20091111:test</include> <include>commons-codec:commons-codec:${commons-codec.vespa.version}:test</include> + <include>de.kherud:llama:${kherud.llama.vespa.version}:test</include> <include>io.airlift:aircompressor:${aircompressor.vespa.version}:test</include> <include>io.airlift:airline:${airline.vespa.version}:test</include> <include>io.prometheus:simpleclient:${prometheus.client.vespa.version}:test</include> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java index e0ddf16ab5f..7991e1c34b6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -47,7 +46,7 @@ public class AnnotatedClusterState implements Cloneable { Map<Node, NodeStateReason> nodeStateReasons) { this.clusterState = Objects.requireNonNull(clusterState, "Cluster state cannot be null"); this.clusterStateReason = Objects.requireNonNull(clusterStateReason, "Cluster state reason cannot be null"); - this.nodeStateReasons = Objects.requireNonNull(nodeStateReasons, "Node state reasons cannot be null"); + this.nodeStateReasons = Map.copyOf(Objects.requireNonNull(nodeStateReasons, "Node state reasons cannot be null")); } public static AnnotatedClusterState emptyState() { @@ -59,7 +58,7 @@ public class AnnotatedClusterState implements Cloneable { } static Map<Node, NodeStateReason> emptyNodeStateReasons() { - return Collections.emptyMap(); + return Map.of(); } public ClusterState getClusterState() { @@ -67,7 +66,7 @@ public class AnnotatedClusterState implements Cloneable { } public Map<Node, NodeStateReason> getNodeStateReasons() { - return Collections.unmodifiableMap(nodeStateReasons); + return nodeStateReasons; } public Optional<ClusterStateReason> getClusterStateReason() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java index 53a39968720..de56272d0e8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java @@ -3,8 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -56,7 +55,7 @@ public class ClusterStateBundle { public FeedBlock(boolean blockFeedInCluster, String description) { this.blockFeedInCluster = blockFeedInCluster; this.description = description; - this.concreteExhaustions = Collections.emptySet(); + this.concreteExhaustions = Set.of(); } public FeedBlock(boolean blockFeedInCluster, String description, @@ -138,7 +137,7 @@ public class ClusterStateBundle { } public Builder bucketSpaces(String... bucketSpaces) { - return bucketSpaces(new TreeSet<>(Arrays.asList(bucketSpaces))); + return bucketSpaces(new TreeSet<>(List.of(bucketSpaces))); } public Builder explicitDerivedStates(Map<String, AnnotatedClusterState> derivedStates) { @@ -165,14 +164,10 @@ public class ClusterStateBundle { return ClusterStateBundle.ofBaselineOnly(baselineState, feedBlock, deferredActivation); } Map<String, AnnotatedClusterState> derived; - if (explicitDerivedStates != null) { - derived = explicitDerivedStates; - } else { - derived = bucketSpaces.stream() - .collect(Collectors.toMap( - Function.identity(), - s -> stateDeriver.derivedFrom(baselineState, s))); - } + derived = Objects.requireNonNullElseGet(explicitDerivedStates, () -> bucketSpaces.stream() + .collect(Collectors.toUnmodifiableMap( + Function.identity(), + s -> stateDeriver.derivedFrom(baselineState, s)))); return new ClusterStateBundle(baselineState, derived, feedBlock, deferredActivation); } } @@ -186,7 +181,7 @@ public class ClusterStateBundle { FeedBlock feedBlock, boolean deferredActivation) { this.baselineState = baselineState; - this.derivedBucketSpaceStates = Collections.unmodifiableMap(derivedBucketSpaceStates); + this.derivedBucketSpaceStates = Map.copyOf(derivedBucketSpaceStates); this.feedBlock = feedBlock; this.deferredActivation = deferredActivation; } @@ -209,11 +204,11 @@ public class ClusterStateBundle { public static ClusterStateBundle ofBaselineOnly(AnnotatedClusterState baselineState, FeedBlock feedBlock, boolean deferredActivation) { - return new ClusterStateBundle(baselineState, Collections.emptyMap(), feedBlock, deferredActivation); + return new ClusterStateBundle(baselineState, Map.of(), feedBlock, deferredActivation); } public static ClusterStateBundle ofBaselineOnly(AnnotatedClusterState baselineState) { - return new ClusterStateBundle(baselineState, Collections.emptyMap()); + return new ClusterStateBundle(baselineState, Map.of()); } public static ClusterStateBundle empty() { @@ -238,7 +233,7 @@ public class ClusterStateBundle { AnnotatedClusterState clonedBaseline = baselineState.cloneWithClusterState( mapper.apply(baselineState.getClusterState().clone())); Map<String, AnnotatedClusterState> clonedDerived = derivedBucketSpaceStates.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().cloneWithClusterState( + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().cloneWithClusterState( mapper.apply(e.getValue().getClusterState().clone())))); return new ClusterStateBundle(clonedBaseline, clonedDerived, feedBlock, deferredActivation); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index 5c29228b858..d57f32ee3cb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -9,7 +9,6 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -174,8 +173,8 @@ public class EventDiffCalculator { private static void emitNodeResourceExhaustionEvents(PerStateParams params, List<Event> events, ContentCluster cluster) { // Feed block events are not ordered by node - Set<NodeResourceExhaustion> fromBlockSet = params.feedBlockFrom != null ? params.feedBlockFrom.getConcreteExhaustions() : Collections.emptySet(); - Set<NodeResourceExhaustion> toBlockSet = params.feedBlockTo != null ? params.feedBlockTo.getConcreteExhaustions() : Collections.emptySet(); + Set<NodeResourceExhaustion> fromBlockSet = params.feedBlockFrom != null ? params.feedBlockFrom.getConcreteExhaustions() : Set.of(); + Set<NodeResourceExhaustion> toBlockSet = params.feedBlockTo != null ? params.feedBlockTo.getConcreteExhaustions() : Set.of(); for (var ex : setSubtraction(fromBlockSet, toBlockSet)) { var info = cluster.getNodeInfo(ex.node); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index a0efaa70b58..44b2c8833db 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import java.time.Duration; import java.util.Collection; -import java.util.Collections; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -433,7 +432,7 @@ public class FleetControllerOptions { private double minMergeCompletionRatio = 1.0; private int maxDivergentNodesPrintedInTaskErrorMessages = 10; private boolean clusterFeedBlockEnabled = false; - private Map<String, Double> clusterFeedBlockLimit = Collections.emptyMap(); + private Map<String, Double> clusterFeedBlockLimit = Map.of(); private double clusterFeedBlockNoiseLevel = 0.01; private int maxNumberOfGroupsAllowedToBeDown = 1; private Function<FleetControllerContext, DatabaseFactory> dbFactoryFn = ZooKeeperDatabaseFactory::new; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java index 200a5564c64..4ed1ab967fd 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.Map; import java.util.Objects; @@ -60,7 +59,7 @@ public class ResourceExhaustionCalculator { this.feedBlockEnabled = feedBlockEnabled; this.feedBlockLimits = feedBlockLimits; this.feedBlockNoiseLevel = 0.0; - this.previouslyBlockedNodeResources = Collections.emptySet(); + this.previouslyBlockedNodeResources = Set.of(); } public ResourceExhaustionCalculator(boolean feedBlockEnabled, Map<String, Double> feedBlockLimits, @@ -74,7 +73,7 @@ public class ResourceExhaustionCalculator { .map(ex -> NodeAndResourceType.of(ex.node.getIndex(), ex.resourceType)) .collect(Collectors.toSet()); } else { - this.previouslyBlockedNodeResources = Collections.emptySet(); + this.previouslyBlockedNodeResources = Set.of(); } } @@ -124,12 +123,12 @@ public class ResourceExhaustionCalculator { effectiveLimit, nodeInfo.getRpcAddress())); } } - return (exceedingLimit != null) ? exceedingLimit : Collections.emptySet(); + return (exceedingLimit != null) ? exceedingLimit : Set.of(); } public Set<NodeResourceExhaustion> enumerateNodeResourceExhaustions(NodeInfo nodeInfo) { if (!nodeInfo.isStorage()) { - return Collections.emptySet(); + return Set.of(); } return resourceExhaustionsFromHostInfo(nodeInfo, nodeInfo.getHostInfo()); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java index 12db5c27375..3a6b9a635fa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlimeClusterStateBundleCodec.java @@ -41,11 +41,10 @@ public class SlimeClusterStateBundleCodec implements ClusterStateBundleCodec, En // TODO add another function that is not toString for this..! states.setString("baseline", stateBundle.getBaselineClusterState().toString()); Cursor spaces = states.setObject("spaces"); - stateBundle.getDerivedBucketSpaceStates().entrySet() - .forEach(entry -> spaces.setString(entry.getKey(), entry.getValue().toString())); + stateBundle.getDerivedBucketSpaceStates().forEach((key, value) -> spaces.setString(key, value.toString())); // Only bother to encode feed block state if cluster is actually blocked - if (stateBundle.getFeedBlock().map(fb -> fb.blockFeedInCluster()).orElse(false)) { + if (stateBundle.getFeedBlock().map(ClusterStateBundle.FeedBlock::blockFeedInCluster).orElse(false)) { Cursor feedBlock = root.setObject("feed-block"); feedBlock.setBool("block-feed-in-cluster", true); feedBlock.setString("description", stateBundle.getFeedBlock().get().getDescription()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleUtil.java index 7c39135291c..8280706bcd2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleUtil.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleUtil.java @@ -14,13 +14,13 @@ public class ClusterStateBundleUtil { public static ClusterStateBundle.Builder makeBundleBuilder(String baselineState, StateMapping... bucketSpaceStates) { return ClusterStateBundle.builder(AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString(baselineState))) - .explicitDerivedStates(Stream.of(bucketSpaceStates).collect(Collectors.toMap(sm -> sm.bucketSpace, + .explicitDerivedStates(Stream.of(bucketSpaceStates).collect(Collectors.toUnmodifiableMap(sm -> sm.bucketSpace, sm -> AnnotatedClusterState.withoutAnnotations(sm.state)))); } public static ClusterStateBundle makeBundle(String baselineState, StateMapping... bucketSpaceStates) { return ClusterStateBundle.of(AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString(baselineState)), - Stream.of(bucketSpaceStates).collect(Collectors.toMap(sm -> sm.bucketSpace, + Stream.of(bucketSpaceStates).collect(Collectors.toUnmodifiableMap(sm -> sm.bucketSpace, sm -> AnnotatedClusterState.withoutAnnotations(sm.state)))); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java index 175b7de8f4f..8048e77b05c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java @@ -11,7 +11,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Collections; import java.util.Map; import java.util.TreeMap; @@ -53,7 +52,7 @@ public class ContentClusterHtmlRendererTest { statsAggregator, 1.0, 10, - Collections.emptyMap(), + Map.of(), eventLog, "pathPrefix", "name"); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index ee0506070b5..3b26e3b6965 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -28,7 +28,6 @@ import org.junit.jupiter.api.AfterEach; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -221,7 +220,7 @@ public abstract class FleetControllerTest implements Waiter { } static Set<Integer> asIntSet(Integer... idx) { - return new HashSet<>(Arrays.asList(idx)); + return new HashSet<>(List.of(idx)); } static Set<ConfiguredNode> asConfiguredNodes(Set<Integer> indices) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java index cab67c0b498..17f58ff40f0 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java @@ -5,7 +5,9 @@ import com.yahoo.document.FixedBucketSpaces; import com.yahoo.vdslib.state.ClusterState; import org.junit.jupiter.api.Test; + import java.util.Arrays; +import java.util.List; import static com.yahoo.vespa.clustercontroller.core.NodeStateReason.MAY_HAVE_MERGES_PENDING; import static com.yahoo.vespa.clustercontroller.core.NodeStateReason.NODE_TOO_UNSTABLE; @@ -110,8 +112,8 @@ public class MaintenanceWhenPendingGlobalMergesTest { @Test void node_with_pending_merges_only_set_to_maintenance_if_eligible() { Fixture f = new Fixture(); - Arrays.asList(1, 2, 3).forEach(idx -> when(f.mockPendingChecker.mayHaveMergesPending(globalSpace(), idx)).thenReturn(true)); - Arrays.asList(1, 2, 4).forEach(idx -> when(f.mockTransitionConstraint.maintenanceTransitionAllowed(idx)).thenReturn(false)); + List.of(1, 2, 3).forEach(idx -> when(f.mockPendingChecker.mayHaveMergesPending(globalSpace(), idx)).thenReturn(true)); + List.of(1, 2, 4).forEach(idx -> when(f.mockTransitionConstraint.maintenanceTransitionAllowed(idx)).thenReturn(false)); AnnotatedClusterState derived = f.deriver.derivedFrom(stateFromString("distributor:5 storage:5"), defaultSpace()); assertThat(derived, equalTo(AnnotatedClusterStateBuilder.ofState("distributor:5 storage:5 .3.s:m") .reason(MAY_HAVE_MERGES_PENDING, 3).build())); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java index b4246806079..168fe6521d5 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; import org.junit.jupiter.api.Test; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -46,20 +46,20 @@ public class ResourceUsageStatsTest { @Test void nodes_above_limit_is_zero_without_feed_block_status() { - var stats = ResourceUsageStats.calculateFrom(Collections.emptyList(), Collections.emptyMap(), Optional.empty()); + var stats = ResourceUsageStats.calculateFrom(List.of(), Map.of(), Optional.empty()); assertEquals(0, stats.getNodesAboveLimit()); } @Test void nodes_above_limit_is_equal_to_node_resource_exhaustions() { - var stats = ResourceUsageStats.calculateFrom(Collections.emptyList(), Collections.emptyMap(), + var stats = ResourceUsageStats.calculateFrom(List.of(), Map.of(), createFeedBlock(exhaustion(1, "disk"), exhaustion(2, "memory"))); assertEquals(2, stats.getNodesAboveLimit()); } @Test void nodes_above_limit_counts_each_node_only_once() { - var stats = ResourceUsageStats.calculateFrom(Collections.emptyList(), Collections.emptyMap(), + var stats = ResourceUsageStats.calculateFrom(List.of(), Map.of(), createFeedBlock(exhaustion(1, "disk"), exhaustion(1, "memory"))); assertEquals(1, stats.getNodesAboveLimit()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java index 9e7665e65ee..bf8ba7756e2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.clustercontroller.core.DummyVdsNode; import com.yahoo.vespa.clustercontroller.core.FleetController; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -62,13 +61,13 @@ public interface Waiter { return waitForState(state, data.getTimeout()); } public ClusterState waitForStateInAllSpaces(String state) { - return waitForState(state, data.getTimeout(), true, Collections.emptySet()); + return waitForState(state, data.getTimeout(), true, Set.of()); } public ClusterState waitForStateInSpace(String space, String state) { - return waitForState(state, data.getTimeout(), false, Collections.singleton(space)); + return waitForState(state, data.getTimeout(), false, Set.of(space)); } public ClusterState waitForState(String state, Duration timeoutMS) { - return waitForState(state, timeoutMS, false, Collections.emptySet()); + return waitForState(state, timeoutMS, false, Set.of()); } public ClusterState waitForStableSystem() { return waitForStableSystem(data.getDummyNodes().size() / 2); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java index 990e11cae15..6dbf0e4bbac 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/Bundle.java @@ -56,7 +56,7 @@ public class Bundle { if (!bundleDir.isDirectory()) { return new ArrayList<>(); } - return Arrays.asList(bundleDir.listFiles((dir, name) -> name.endsWith(".jar"))); + return List.of(bundleDir.listFiles((dir, name) -> name.endsWith(".jar"))); } public List<DefEntry> getDefEntries() { diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 7d8f827bf31..28bf8e10a93 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -55,7 +55,6 @@ import java.io.StringReader; import java.nio.file.Files; import java.security.MessageDigest; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -435,11 +434,11 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { File sdDir = applicationFile(appDir, SEARCH_DEFINITIONS_DIR.getRelative()); if (sdDir.isDirectory()) - schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> validSchemaFilename(name)))); + schemaFiles.addAll(List.of(sdDir.listFiles((dir, name) -> validSchemaFilename(name)))); sdDir = applicationFile(appDir, SCHEMAS_DIR.getRelative()); if (sdDir.isDirectory()) - schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> validSchemaFilename(name)))); + schemaFiles.addAll(List.of(sdDir.listFiles((dir, name) -> validSchemaFilename(name)))); return schemaFiles; } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index 39c2890c30c..27041f8db39 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -12,8 +12,8 @@ import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import static com.yahoo.foo.StructtypesConfig.Simple.Gender.Enum.FEMALE; @@ -176,7 +176,7 @@ public class ConfigInstanceBuilderTest { doublearr(123.0). stringarr("bar"). enumarr(Enumarr.VALUES). - refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + refarr(List.of(":parent:", ":parent", "parent:")). // test collection based setter fileArr("bin"). intMap("one", 1). intMap("two", 2). diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index c50656eac12..2ac679aa9ab 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -6,7 +6,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; -import java.util.Arrays; +import java.util.List; import java.util.Optional; import static com.yahoo.test.FunctionTestConfig.BasicStruct; @@ -136,7 +136,7 @@ public class ConfigInstanceEqualsTest { doublearr(123.0). stringarr("bar"). enumarr(Enumarr.VALUES). - refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + refarr(List.of(":parent:", ":parent", "parent:")). // test collection based setter fileArr("bin"). urlArr(new UrlReference("http://docs.vespa.ai")). modelArr(ModelReference.unresolved(Optional.empty(), diff --git a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java index a9151692bb5..f8d16fa7ac3 100644 --- a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java +++ b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java @@ -3,7 +3,6 @@ package com.yahoo.config; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -73,7 +72,7 @@ public class NodeVectorTest { assertTrue(v.contains(val)); assertFalse(v.contains(barNode())); assertTrue(v.contains(val)); - assertFalse(v.containsAll(Arrays.asList(val, barNode()))); + assertFalse(v.containsAll(List.of(val, barNode()))); } @Test @@ -114,7 +113,7 @@ public class NodeVectorTest { private static class TestNodeVector extends LeafNodeVector<String, StringNode> { TestNodeVector(String... values) { - super(Arrays.asList(values), new StringNode()); + super(List.of(values), new StringNode()); } } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 9a975f1b727..42e7e23dfcc 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1324,7 +1324,9 @@ "public java.lang.String unknownConfigDefinition()", "public int searchHandlerThreadpool()", "public boolean alwaysMarkPhraseExpensive()", - "public boolean sortBlueprintsByCost()" + "public boolean sortBlueprintsByCost()", + "public int persistenceThreadMaxFeedOpBatchSize()", + "public boolean logserverOtelCol()" ], "fields" : [ ] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0f89608f9bd..deef16cfe8c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -14,7 +14,6 @@ import java.io.File; import java.io.IOException; import java.io.Reader; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -213,11 +212,11 @@ public interface ApplicationPackage { } default Map<Version, FileRegistry> getFileRegistries() { - return Collections.emptyMap(); + return Map.of(); } default Map<String, String> legacyOverrides() { - return Collections.emptyMap(); + return Map.of(); } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 16cd7bf7e88..4308e0c2a0e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -386,7 +386,7 @@ public class DeploymentSpec { public abstract boolean concerns(Environment environment, Optional<RegionName> region); /** Returns the zones deployed to in this step. */ - public List<DeclaredZone> zones() { return Collections.emptyList(); } + public List<DeclaredZone> zones() { return List.of(); } /** The delay introduced by this step (beyond the time it takes to execute the step). */ public Duration delay() { return Duration.ZERO; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java index 9eef386d7d0..08fb4d75183 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Notifications.java @@ -4,14 +4,11 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; -import static java.util.Collections.emptyList; - /** * Configuration of notifications for deployment jobs. * @@ -25,7 +22,7 @@ import static java.util.Collections.emptyList; */ public class Notifications { - private static final Notifications none = new Notifications(Collections.emptyMap(), Collections.emptyMap()); + private static final Notifications none = new Notifications(Map.of(), Map.of()); public static Notifications none() { return none; } private final Map<When, List<String>> emailAddresses; @@ -60,7 +57,7 @@ public class Notifications { /** Returns all email addresses to notify for the given condition. */ public Set<String> emailAddressesFor(When when) { ImmutableSet.Builder<String> addresses = ImmutableSet.builder(); - addresses.addAll(emailAddresses.getOrDefault(when, emptyList())); + addresses.addAll(emailAddresses.getOrDefault(when, List.of())); for (When include : when.includes) addresses.addAll(emailAddressesFor(include)); return addresses.build(); @@ -69,7 +66,7 @@ public class Notifications { /** Returns all roles for which email notification is to be sent for the given condition. */ public Set<Role> emailRolesFor(When when) { ImmutableSet.Builder<Role> roles = ImmutableSet.builder(); - roles.addAll(emailRoles.getOrDefault(when, emptyList())); + roles.addAll(emailRoles.getOrDefault(when, List.of())); for (When include : when.includes) roles.addAll(emailRolesFor(include)); return roles.build(); @@ -82,17 +79,17 @@ public class Notifications { author; public static String toValue(Role role) { - switch (role) { - case author: return "author"; - default: throw new IllegalArgumentException("Unexpected constant '" + role.name() + "'."); + if (Objects.requireNonNull(role) == Role.author) { + return "author"; } + throw new IllegalArgumentException("Unexpected constant '" + role.name() + "'."); } public static Role fromValue(String value) { - switch (value) { - case "author": return author; - default: throw new IllegalArgumentException("Unknown value '" + value + "'."); + if (value.equals("author")) { + return author; } + throw new IllegalArgumentException("Unknown value '" + value + "'."); } } @@ -109,15 +106,15 @@ public class Notifications { private final List<When> includes; When(When... includes) { - this.includes = Arrays.asList(includes); + this.includes = List.of(includes); } public static String toValue(When when) { - switch (when) { - case failing: return "failing"; - case failingCommit: return "failing-commit"; - default: throw new IllegalArgumentException("Unexpected constant '" + when.name() + "'."); - } + return switch (when) { + case failing -> "failing"; + case failingCommit -> "failing-commit"; + default -> throw new IllegalArgumentException("Unexpected constant '" + when.name() + "'."); + }; } public static When fromValue(String value) { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 4a4d4648deb..ad5ab5b7ee4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -112,6 +112,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default int searchHandlerThreadpool() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean alwaysMarkPhraseExpensive() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean sortBlueprintsByCost() { return false; } + @ModelFeatureFlag(owners = {"vekterli"}) default int persistenceThreadMaxFeedOpBatchSize() { return 1; } + @ModelFeatureFlag(owners = {"olaa"}) default boolean logserverOtelCol() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java index 3424dad4ee5..f6bc8c34aa6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -23,7 +22,7 @@ public class SuperModel { private final boolean complete; public SuperModel() { - this(Collections.emptyMap(), false); + this(Map.of(), false); } public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) { diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 99c9dad02de..05807ae6cc1 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -23,7 +23,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -1152,7 +1151,7 @@ public class DeploymentSpecTest { <instance id='default'/> </deployment> """); - assertEquals(Collections.emptyList(), spec.requireInstance("default").endpoints()); + assertEquals(List.of(), spec.requireInstance("default").endpoints()); assertEquals(ZoneEndpoint.defaultEndpoint, spec.zoneEndpoint(InstanceName.defaultName(), defaultId(), ClusterSpec.Id.from("cluster"))); diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index e102e3afaa7..e25069fdc3a 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -16,7 +16,6 @@ import java.io.StringReader; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -599,7 +598,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void emptyEndpoints() { var spec = DeploymentSpec.fromXml("<deployment><endpoints/></deployment>"); - assertEquals(Collections.emptyList(), spec.requireInstance("default").endpoints()); + assertEquals(List.of(), spec.requireInstance("default").endpoints()); } @Test diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/HostInfoTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/HostInfoTest.java index 6bd94d0aa74..7e4b0a07952 100644 --- a/config-model-api/src/test/java/com/yahoo/config/model/api/HostInfoTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/HostInfoTest.java @@ -2,7 +2,8 @@ package com.yahoo.config.model.api; import org.junit.Test; -import java.util.Arrays; + +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -10,10 +11,10 @@ import static org.junit.Assert.assertNotEquals; public class HostInfoTest { @Test public void testEquals() { - HostInfo a = new HostInfo("foo.yahoo.com", Arrays.asList(new ServiceInfo("foo", "bar", null, null, "config-id", "host-name"))); - HostInfo b = new HostInfo("foo.yahoo.com", Arrays.asList(new ServiceInfo("foo", "bar", null, null, "config-id", "host-name"))); - HostInfo c = new HostInfo("foo.yahoo.com", Arrays.asList(new ServiceInfo("foo", "baz", null, null, "config-id", "host-name"))); - HostInfo d = new HostInfo("foo.yahoo.com", Arrays.asList(new ServiceInfo("bar", "baz", null, null, "config-id", "host-name"))); + HostInfo a = new HostInfo("foo.yahoo.com", List.of(new ServiceInfo("foo", "bar", null, null, "config-id", "host-name"))); + HostInfo b = new HostInfo("foo.yahoo.com", List.of(new ServiceInfo("foo", "bar", null, null, "config-id", "host-name"))); + HostInfo c = new HostInfo("foo.yahoo.com", List.of(new ServiceInfo("foo", "baz", null, null, "config-id", "host-name"))); + HostInfo d = new HostInfo("foo.yahoo.com", List.of(new ServiceInfo("bar", "baz", null, null, "config-id", "host-name"))); HostInfo e = new HostInfo("bar.yahoo.com", null); assertEquals(a, b); assertNotEquals(a, c); diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/PortInfoTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/PortInfoTest.java index 08e00e5c771..3ee9217e67f 100644 --- a/config-model-api/src/test/java/com/yahoo/config/model/api/PortInfoTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/PortInfoTest.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.api; import org.junit.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -11,10 +11,10 @@ import static org.junit.Assert.assertNotEquals; public class PortInfoTest { @Test public void testEquals() { - PortInfo a = new PortInfo(1234, Arrays.asList("foo")); - PortInfo b = new PortInfo(1234, Arrays.asList("foo")); - PortInfo c = new PortInfo(1234, Arrays.asList("foo", "bar")); - PortInfo d = new PortInfo(12345, Arrays.asList("foo")); + PortInfo a = new PortInfo(1234, List.of("foo")); + PortInfo b = new PortInfo(1234, List.of("foo")); + PortInfo c = new PortInfo(1234, List.of("foo", "bar")); + PortInfo d = new PortInfo(12345, List.of("foo")); assertEquals(a, b); assertNotEquals(a, c); assertNotEquals(a, d); diff --git a/config-model-api/src/test/java/com/yahoo/config/model/api/ServiceInfoTest.java b/config-model-api/src/test/java/com/yahoo/config/model/api/ServiceInfoTest.java index f1824763c12..b249b82d818 100644 --- a/config-model-api/src/test/java/com/yahoo/config/model/api/ServiceInfoTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/model/api/ServiceInfoTest.java @@ -3,8 +3,8 @@ package com.yahoo.config.model.api; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -17,14 +17,14 @@ public class ServiceInfoTest { String commonConfigId = "common-config-id"; String commonHostName = "common-host"; - ServiceInfo a = new ServiceInfo("0", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), commonConfigId, commonHostName); - ServiceInfo b = new ServiceInfo("0", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), commonConfigId, commonHostName); - ServiceInfo c = new ServiceInfo("0", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "baz"), commonConfigId, commonHostName); - ServiceInfo d = new ServiceInfo("0", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("bar", "bar"), commonConfigId, commonHostName); - ServiceInfo e = new ServiceInfo("0", "1", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), commonConfigId, commonHostName); - ServiceInfo f = new ServiceInfo("1", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), commonConfigId, commonHostName); - ServiceInfo g = new ServiceInfo("1", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), "different-config-id", commonHostName); - ServiceInfo h = new ServiceInfo("1", "0", Arrays.asList(new PortInfo(33, null)), Collections.singletonMap("foo", "bar"), commonConfigId, "different-host"); + ServiceInfo a = new ServiceInfo("0", "0", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), commonConfigId, commonHostName); + ServiceInfo b = new ServiceInfo("0", "0", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), commonConfigId, commonHostName); + ServiceInfo c = new ServiceInfo("0", "0", List.of(new PortInfo(33, null)), Map.of("foo", "baz"), commonConfigId, commonHostName); + ServiceInfo d = new ServiceInfo("0", "0", List.of(new PortInfo(33, null)), Map.of("bar", "bar"), commonConfigId, commonHostName); + ServiceInfo e = new ServiceInfo("0", "1", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), commonConfigId, commonHostName); + ServiceInfo f = new ServiceInfo("1", "0", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), commonConfigId, commonHostName); + ServiceInfo g = new ServiceInfo("1", "0", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), "different-config-id", commonHostName); + ServiceInfo h = new ServiceInfo("1", "0", List.of(new PortInfo(33, null)), Map.of("foo", "bar"), commonConfigId, "different-host"); assertEquals(a, b); assertNotEquals(a, c); diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index d9d7dd9eaf0..f324ceef5ab 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -233,6 +233,37 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } } + // add cluster type? + // add cluster name? + public record StatePortInfo(String hostName, int portNumber, + String serviceName, String serviceType) + {} + + public List<StatePortInfo> getStatePorts() { + List<StatePortInfo> result = new ArrayList<>(); + for (HostResource modelHost : hostSystem().getHosts()) { + String hostName = modelHost.getHostname(); + for (Service modelService : modelHost.getServices()) { + String serviceName = modelService.getServiceName(); + String serviceType = modelService.getServiceType(); + PortsMeta portsMeta = modelService.getPortsMeta(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + int portNumber = modelService.getRelativePort(i); + boolean hasState = false; + boolean isHttp = false; + for (String tag : portsMeta.getTagsAt(i)) { + if (tag.equals("state")) hasState = true; + if (tag.equals("http")) isHttp = true; + } + if (hasState && isHttp) { + result.add(new StatePortInfo(hostName, portNumber, serviceName, serviceType)); + } + } + } + } + return result; + } + private List<Services.Builder> getServices(HostResource modelHost) { List<Services.Builder> ret = new ArrayList<>(); for (Service modelService : modelHost.getServices()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java index 51bd01de5bc..f25f130eed0 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import java.util.Collection; -import java.util.Collections; +import java.util.List; /** * A resolver of implementations of named config models. @@ -44,7 +44,7 @@ public abstract class ConfigModelRegistry { @Override public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { - return Collections.emptyList(); + return List.of(); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java index 8fe1372ef2f..7242d3de1dd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -58,7 +57,7 @@ public class MapConfigModelRegistry extends ConfigModelRegistry { @SafeVarargs @SuppressWarnings("varargs") public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) { - return new MapConfigModelRegistry(Arrays.asList(builders)); + return new MapConfigModelRegistry(List.of(builders)); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index f19341098f4..57a75bd8a38 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -50,7 +50,6 @@ import java.io.Reader; import java.io.UncheckedIOException; import java.time.Instant; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -329,7 +328,7 @@ public class DeployState implements ConfigDefinitionStore { private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); private Optional<Model> previousModel = Optional.empty(); private Set<ContainerEndpoint> endpoints = Set.of(); - private Collection<MlModelImporter> modelImporters = Collections.emptyList(); + private Collection<MlModelImporter> modelImporters = List.of(); private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 11d3a48ee51..3c45588a054 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import java.net.URI; import java.security.cert.X509Certificate; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -35,10 +34,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean multitenant = false; private ApplicationId applicationId = ApplicationId.defaultId(); - private List<ConfigServerSpec> configServerSpecs = Collections.emptyList(); + private List<ConfigServerSpec> configServerSpecs = List.of(); private boolean hostedVespa = false; private Zone zone = Zone.defaultZone(); - private final Set<ContainerEndpoint> endpoints = Collections.emptySet(); + private final Set<ContainerEndpoint> endpoints = Set.of(); private boolean useDedicatedNodeForLogserver = false; private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; @@ -55,10 +54,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; - private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); + private List<TenantSecretStore> tenantSecretStores = List.of(); private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; - private List<X509Certificate> operatorCertificates = Collections.emptyList(); + private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; private double resourceLimitMemory = 0.8; private double minNodeRatioPerGroup = 0.0; @@ -83,6 +82,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowUserFilters = true; private List<DataplaneToken> dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; + private int persistenceThreadMaxFeedOpBatchSize = 1; + private boolean logserverOtelCol = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -139,6 +140,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean allowUserFilters() { return allowUserFilters; } @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } + @Override public boolean logserverOtelCol() { return logserverOtelCol; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -369,6 +372,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setPersistenceThreadMaxFeedOpBatchSize(int maxBatchSize) { + this.persistenceThreadMaxFeedOpBatchSize = maxBatchSize; + return this; + } + + public TestProperties setLogserverOtelCol(boolean logserverOtelCol) { + this.logserverOtelCol = logserverOtelCol; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java index e411a8e3e77..dcf1c8629a4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java @@ -135,7 +135,7 @@ public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstance } // For collections, we don't require that dependency has been added, we just give an empty collection if (isCollection(param)) - return Collections.emptyList(); + return List.of(); throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 5d3db7f676a..365543a549b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -31,7 +31,6 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -143,7 +142,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { - return Collections.emptyMap(); + return Map.of(); } @Override @@ -203,7 +202,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { - return Collections.emptyList(); + return List.of(); } public QueryProfileRegistry getQueryProfiles() { return queryProfileRegistry; } @@ -240,7 +239,7 @@ public class MockApplicationPackage implements ApplicationPackage { private File root = new File("nonexisting"); private String hosts = null; private String services = null; - private List<String> schemas = Collections.emptyList(); + private List<String> schemas = List.of(); private Map<Path, MockApplicationFile> files = new LinkedHashMap<>(); private String schemaDir = null; private String deploymentSpec = null; @@ -277,12 +276,12 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withSearchDefinition(String searchDefinition) { - this.schemas = Collections.singletonList(searchDefinition); + this.schemas = List.of(searchDefinition); return this; } public Builder withSchemas(List<String> searchDefinition) { - this.schemas = Collections.unmodifiableList(searchDefinition); + this.schemas = List.copyOf(searchDefinition); return this; } @@ -369,8 +368,8 @@ public class MockApplicationPackage implements ApplicationPackage { } private List<NamedReader> asNamedReaderList(String value) { - if (value == null) return Collections.emptyList(); - return Collections.singletonList(new NamedReader(extractId(value) + ".xml", new StringReader(value))); + if (value == null) return List.of(); + return List.of(new NamedReader(extractId(value) + ".xml", new StringReader(value))); } private String extractId(String xmlStringWithIdAttribute) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index 94969aa5324..509c10c7fc9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import org.w3c.dom.Element; import java.io.StringReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -20,7 +19,7 @@ public class TestUtil { public static Element parse(String... xmlLines) { List<String> lines = new ArrayList<>(); lines.add("<?xml version='1.0' encoding='utf-8' ?>"); - lines.addAll(Arrays.asList(xmlLines)); + lines.addAll(List.of(xmlLines)); try { return XmlHelper.getDocument(new StringReader(CollectionUtil.mkString(lines, "\n").replace("'", "\""))).getDocumentElement(); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 77e123fefef..c5140dd6fc8 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptySet; /** * @author baldersheim @@ -47,7 +46,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final Set<String> importedFieldNames; public NewDocumentType(Name name) { - this(name, emptySet()); + this(name, Set.of()); } public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { @@ -60,7 +59,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } public NewDocumentType(Name name, Set<Name> documentReferences) { - this(name, documentReferences, emptySet()); + this(name, documentReferences, Set.of()); } public NewDocumentType(Name name, @@ -74,8 +73,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp if (fs != null) { this.fieldSets.addAll(fs.userFieldSets().values()); for (FieldSet f : fs.builtInFieldSets().values()) { - if ((f.getName() != BuiltInFieldSets.INTERNAL_FIELDSET_NAME) && - (f.getName() != BuiltInFieldSets.SEARCH_FIELDSET_NAME)) { + if (!BuiltInFieldSets.INTERNAL_FIELDSET_NAME.equals(f.getName()) && + !BuiltInFieldSets.SEARCH_FIELDSET_NAME.equals(f.getName())) { fieldSets.add(f); } } diff --git a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index 2e964ac3624..749cdbfd8b7 100644 --- a/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -351,7 +350,7 @@ public class ApplicationBuilder { RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), + return createFromFiles(List.of(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, queryprofileRegistry); } @@ -414,8 +413,8 @@ public class ApplicationBuilder { queryProfileRegistry); var fnli = Files.list(new File(dir).toPath()) - .map(p -> p.toString()) - .filter(fn -> AbstractApplicationPackage.validSchemaFilename(fn)) + .map(java.nio.file.Path::toString) + .filter(AbstractApplicationPackage::validSchemaFilename) .sorted(); for (var i = fnli.iterator(); i.hasNext(); ) { builder.addSchemaFile(i.next()); diff --git a/config-model/src/main/java/com/yahoo/schema/FieldSets.java b/config-model/src/main/java/com/yahoo/schema/FieldSets.java index 1dbf496992b..6464bd1681e 100644 --- a/config-model/src/main/java/com/yahoo/schema/FieldSets.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldSets.java @@ -47,11 +47,7 @@ public class FieldSets { * @param field field to add to field set */ public void addBuiltInFieldSetItem(String setName, String field) { - if (builtInFieldSets.get(setName) == null) { - // First entry in this set - builtInFieldSets.put(setName, new FieldSet(setName)); - } - builtInFieldSets.get(setName).addFieldName(field); + builtInFieldSets.computeIfAbsent(setName, FieldSet::new).addFieldName(field); } /** Returns the built in field sets, unmodifiable */ diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 22bf1880cd7..82ed45028b3 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -487,12 +486,12 @@ public class RankProfile implements Cloneable { } void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, Collections.emptyList(), rankingExpression), false); + this.firstPhaseRanking = new RankingExpressionFunction(new ExpressionFunction(FIRST_PHASE, List.of(), rankingExpression), false); } public void setFirstPhaseRanking(String expression) { try { - firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, Collections.emptyList(), expression), false); + firstPhaseRanking = new RankingExpressionFunction(parseRankingExpression(FIRST_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); } @@ -515,7 +514,7 @@ public class RankProfile implements Cloneable { public void setSecondPhaseRanking(String expression) { try { - secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, Collections.emptyList(), expression), false); + secondPhaseRanking = new RankingExpressionFunction(parseRankingExpression(SECOND_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -535,7 +534,7 @@ public class RankProfile implements Cloneable { public void setGlobalPhaseRanking(String expression) { try { - globalPhaseRanking = new RankingExpressionFunction(parseRankingExpression(GLOBAL_PHASE, Collections.emptyList(), expression), false); + globalPhaseRanking = new RankingExpressionFunction(parseRankingExpression(GLOBAL_PHASE, List.of(), expression), false); } catch (ParseException e) { throw new IllegalArgumentException("Illegal global-phase ranking function", e); @@ -1036,7 +1035,7 @@ public class RankProfile implements Cloneable { Map<Reference, TensorType> featureTypes = featureTypes(); // Function compiling first pass: compile inline functions without resolving other functions Map<String, RankingExpressionFunction> inlineFunctions = - compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Map.of(), expressionTransforms); firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java index 0fddcbd4cc7..4deee1dce9f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java @@ -4,9 +4,7 @@ package com.yahoo.schema; import com.yahoo.schema.document.SDDocumentType; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +25,7 @@ public class RankProfileRegistry { private static final String globalRankProfilesKey = "[global]"; /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ - static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); + static final Set<String> overridableRankProfileNames = Set.of("default", "unranked"); public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java index 9774868db9c..0c669e29538 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java @@ -7,7 +7,6 @@ import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import java.io.IOException; -import java.util.Collections; import java.util.List; /** @@ -30,7 +29,7 @@ public class Deriver { } public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) { - return getDocumentManagerConfig(Collections.singletonList(sd)); + return getDocumentManagerConfig(List.of(sd)); } public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas) { @@ -38,7 +37,7 @@ public class Deriver { } public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { - return getDocumentTypesConfig(Collections.singletonList(schema)); + return getDocumentTypesConfig(List.of(schema)); } public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> schemas) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index 34ed9f2f60c..677e4b31bdf 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.search.IndexschemaConfig; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -147,7 +146,7 @@ public class IndexSchema extends Derived { static List<Field> flattenField(Field field) { DataType fieldType = field.getDataType(); if (fieldType.getPrimitiveType() != null){ - return Collections.singletonList(field); + return List.of(field); } if (fieldType instanceof ArrayDataType) { List<Field> ret = new LinkedList<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 999d040b48c..ac936ee989e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -5,7 +5,9 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.MapDataType; import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.TensorDataType; @@ -143,8 +145,8 @@ public final class Attribute implements Cloneable, Serializable { } /** Creates an attribute with default settings */ - public Attribute(String name, DataType fieldType) { - this(name, convertDataType(fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); + public Attribute(String schemaName, String fieldName, String name, DataType fieldType) { + this(name, convertDataType(schemaName, fieldName, fieldType), convertCollectionType(fieldType), convertTensorType(fieldType), convertTargetType(fieldType)); setRemoveIfZero(fieldType instanceof WeightedSetDataType wsdt && wsdt.removeIfZero()); setCreateIfNonExistent(fieldType instanceof WeightedSetDataType wsdt && wsdt.createIfNonExistent()); } @@ -266,12 +268,26 @@ public final class Attribute implements Cloneable, Serializable { private void setType(Type type) { this.type=type; } public void setCollectionType(CollectionType type) { this.collectionType=type; } + private static void failDataType(String schemaName, String fieldName, String dataType) throws IllegalArgumentException { + throw new IllegalArgumentException("For schema '" + schemaName + "': Field '" + fieldName + "' of type '" + dataType + "' cannot be an attribute. " + + "Instead specify the struct fields to be searchable as attribute"); + } + public static void validateDataType(String schemaName, String fieldName, DataType fieldType) throws IllegalArgumentException { + if (fieldType instanceof MapDataType mapType) { + failDataType(schemaName, fieldName, "map<" + mapType.getKeyType().getName() + "," + mapType.getValueType().getName() + ">"); + } + if (fieldType instanceof ArrayDataType arrayType && arrayType.getNestedType() instanceof StructDataType nestedType) { + failDataType(schemaName, fieldName, "array<" + nestedType.getName() + ">"); + } + } + /** Converts to the right attribute type from a field datatype */ - public static Type convertDataType(DataType fieldType) { + public static Type convertDataType(String schemaName, String fieldName, DataType fieldType) { + validateDataType(schemaName, fieldName, fieldType); if (fieldType instanceof NewDocumentReferenceDataType) { return Type.REFERENCE; } else if (fieldType instanceof CollectionDataType) { - return convertDataType(((CollectionDataType) fieldType).getNestedType()); + return convertDataType(schemaName, fieldName, ((CollectionDataType) fieldType).getNestedType()); } FieldValue fval = fieldType.createFieldValue(); if (fval instanceof StringFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java index cb98cb79e01..7ade7ada061 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -131,7 +130,7 @@ public class ImmutableImportedSDField implements ImmutableSDField { @Override public Map<String, String> getAliasToName() { - return Collections.emptyMap(); + return Map.of(); } @Override diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 538cb56d210..f165141b16e 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -398,22 +398,23 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */ - public void parseIndexingScript(String script) { - parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + public void parseIndexingScript(String schemaName, String script) { + parseIndexingScript(schemaName, script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); } - public void parseIndexingScript(String script, Linguistics linguistics, Map<String, Embedder> embedders) { + public void parseIndexingScript(String schemaName, String script, Linguistics linguistics, Map<String, Embedder> embedders) { try { ScriptParserContext config = new ScriptParserContext(linguistics, embedders); config.setInputStream(new IndexingInput(script)); - setIndexingScript(ScriptExpression.newInstance(config)); + setIndexingScript(schemaName, ScriptExpression.newInstance(config)); } catch (ParseException e) { throw new IllegalArgumentException("Failed to parse script '" + script + "'", e); } } /** Sets the indexing script of this, or null to not use a script */ - public void setIndexingScript(ScriptExpression exp) { + + public void setIndexingScript(String schemaName, ScriptExpression exp) { if (exp == null) { exp = new ScriptExpression(); } @@ -441,13 +442,13 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { } Attribute attribute = attributes.get(fieldName); if (attribute == null) { - addAttribute(new Attribute(fieldName, getDataType())); + addAttribute(new Attribute(schemaName, fieldName, fieldName, getDataType())); } } }.visit(indexingScript); } for (SDField structField : getStructFields()) { - structField.setIndexingScript(exp); + structField.setIndexingScript(schemaName, exp); } } diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java index 22a9eed2914..5a0c2c3c915 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java @@ -12,7 +12,7 @@ import com.yahoo.schema.document.SDField; public interface FieldOperation extends Comparable<FieldOperation> { /** Apply this operation on the given field */ - void apply(SDField field); + void apply(String schemaName, SDField field); @Override default int compareTo(FieldOperation other) { diff --git a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java index f5366c4b07a..11065f040ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java @@ -28,8 +28,8 @@ public class IndexingOperation implements FieldOperation { public ScriptExpression getScript() { return script; } - public void apply(SDField field) { - field.setIndexingScript(script); + public void apply(String schemaName, SDField field) { + field.setIndexingScript(schemaName, script); } /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index e3ca0090408..7659a1e6562 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -2,6 +2,7 @@ package com.yahoo.schema.parser; import com.yahoo.document.DataType; +import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; import com.yahoo.schema.Index; import com.yahoo.schema.Schema; @@ -49,10 +50,10 @@ public class ConvertParsedFields { (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); } - void convertSorting(SDField field, ParsedSorting parsed, String name) { + void convertSorting(Schema schema, SDField field, ParsedSorting parsed, String name) { Attribute attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } Sorting sorting = attribute.getSorting(); @@ -66,7 +67,7 @@ public class ConvertParsedFields { parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); } - void convertAttribute(SDField field, ParsedAttribute parsed) { + void convertAttribute(Schema schema, SDField field, ParsedAttribute parsed) { String name = parsed.name(); String fieldName = field.getName(); Attribute attribute = null; @@ -76,7 +77,7 @@ public class ConvertParsedFields { if (attribute == null) { attribute = field.getAttributes().get(name); if (attribute == null) { - attribute = new Attribute(name, field.getDataType()); + attribute = new Attribute(schema.getName(), field.getName(), name, field.getDataType()); field.addAttribute(attribute); } } @@ -102,7 +103,7 @@ public class ConvertParsedFields { } var sorting = parsed.getSorting(); if (sorting.isPresent()) { - convertSorting(field, sorting.get(), name); + convertSorting(schema, field, sorting.get(), name); } } @@ -143,19 +144,22 @@ public class ConvertParsedFields { convertMatchSettings(field, parsed.matchSettings()); var indexing = parsed.getIndexing(); if (indexing.isPresent()) { - field.setIndexingScript(indexing.get().script()); + field.setIndexingScript(schema.getName(), indexing.get().script()); + } + if (field.doesAttributing() && !GeoPos.isAnyPos(field.getDataType())) { + Attribute.validateDataType(schema.getName(), field.getName(), field.getDataType()); } parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { - convertAttribute(field, attribute); + convertAttribute(schema, field, attribute); } for (var summaryField : parsed.getSummaryFields()) { var dataType = field.getDataType(); var otherType = summaryField.getType(); if (otherType != null && summaryField.getHasExplicitType()) { - schema.getDeployLogger().log(Level.FINE, () -> "For schema '" + schema.getName() + + schema.getDeployLogger().log(Level.WARNING, () -> "For schema '" + schema.getName() + "', field '" + field.getName() + "', summary '" + summaryField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + @@ -190,7 +194,7 @@ public class ConvertParsedFields { convertCommonFieldSettings(schema, structField, parsed); } - private void convertExtraFieldSettings(SDField field, ParsedField parsed) { + private void convertExtraFieldSettings(Schema schema, SDField field, ParsedField parsed) { String name = parsed.name(); for (var dictOp : parsed.getDictionaryOptions()) { var dictionary = field.getOrSetDictionary(); @@ -208,7 +212,7 @@ public class ConvertParsedFields { field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); } parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); - parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); + parsed.getSorting().ifPresent(sortInfo -> convertSorting(schema, field, sortInfo, name)); if (parsed.hasBolding()) { // TODO must it be so ugly: SummaryField summaryField = field.getSummaryField(name, true); @@ -288,7 +292,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(document, name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); document.addField(field); return field; } @@ -298,7 +302,7 @@ public class ConvertParsedFields { DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(schema.getDocument(), name, dataType); convertCommonFieldSettings(schema, field, parsed); - convertExtraFieldSettings(field, parsed); + convertExtraFieldSettings(schema, field, parsed); schema.addExtraField(field); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index ee15b95b198..3c87044850f 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -233,7 +233,7 @@ public class ConvertParsedSchemas { var parsedType = parsedField.getType(); if (parsedType != null) { var log = schema.getDeployLogger(); - log.log(Level.FINE, () -> "For schema '" + schema.getName() + + log.log(Level.WARNING, () -> "For schema '" + schema.getName() + "', document-summary '" + parsed.name() + "', summary field '" + parsedField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index 67297245ff1..587ffcb86c7 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -6,16 +6,12 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.Field; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; -import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.LinkedHashSet; -import java.util.Set; - /** * This processor creates a {@link com.yahoo.schema.document.SDDocumentType} for each {@link Schema} * object which holds all the data that search @@ -24,8 +20,6 @@ import java.util.Set; */ public class AddExtraFieldsToDocument extends Processor { - Set<String> extraSummaryFields = new LinkedHashSet<String>(); - AddExtraFieldsToDocument(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } @@ -39,31 +33,7 @@ public class AddExtraFieldsToDocument extends Processor { } for (var docsum : schema.getSummaries().values()) { for (var summaryField : docsum.getSummaryFields().values()) { - var transform = summaryField.getTransform(); - if (transform.isDynamic() && DynamicSummaryTransformUtils.summaryFieldIsRequiredInDocumentType(summaryField) || - transform == SummaryTransform.NONE || - transform == SummaryTransform.DOCUMENT_ID) - { - // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. - addSummaryField(schema, document, summaryField, validate); - } else { - // skip: generated from attribute or similar, - // so does not need to be included as an extra - // field in the document type - } - } - } - /* - * Don't use extra summary fields when generating summaries. They will not be created nor populated by - * future vespa versions. When vespa version 'X' stops using these fields and vespa version 'Y' stops - * populating the fields, rollback from vespa version >= 'Y' to vespa version < 'X' will break (missing - * summary fields). - */ - for (var docsum : schema.getSummaries().values()) { - for (var summaryField : docsum.getSummaryFields().values()) { - if (extraSummaryFields.contains(summaryField.getName())) { - considerCopyTransformForExtraSummaryField(schema, summaryField); - } + considerCopyTransformForExtraSummaryField(schema, summaryField); } } } @@ -81,24 +51,6 @@ public class AddExtraFieldsToDocument extends Processor { addField(schema, document, field, validate); } - private void addSummaryField(Schema schema, SDDocumentType document, SummaryField field, boolean validate) { - Field docField = document.getField(field.getName()); - if (docField == null) { - ImmutableSDField existingField = schema.getField(field.getName()); - if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType()); - newField.setIsExtraField(true); - extraSummaryFields.add(field.getName()); - document.addField(newField); - } else if (!existingField.isImportedField()) { - document.addField(existingField.asField()); - } - } else if (!docField.getDataType().equals(field.getDataType())) { - if (validate) - throw newProcessException(schema, field, "Summary field has conflicting type."); - } - } - private void addField(Schema schema, SDDocumentType document, Field field, boolean validate) { if (document.getField(field.getName()) != null && !(document.getField(field.getName()) == field)) { if (validate) @@ -116,6 +68,6 @@ public class AddExtraFieldsToDocument extends Processor { private boolean fieldIsExtraSummaryField(Schema schema, String name) { var field = schema.getConcreteField(name); - return field == null || extraSummaryFields.contains(name); + return field == null; } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index 7bd66ad8f0b..deb57e157f3 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -86,14 +86,14 @@ public class CreatePositionZCurve extends Processor { SummaryTransform.DISTANCE, summaryTo, validate); } // clear indexing script - field.setIndexingScript(null); + field.setIndexingScript(schema.getName(), null); SDField posX = field.getStructField(PositionDataType.FIELD_X); if (posX != null) { - posX.setIndexingScript(null); + posX.setIndexingScript(schema.getName(), null); } SDField posY = field.getStructField(PositionDataType.FIELD_Y); if (posY != null) { - posY.setIndexingScript(null); + posY.setIndexingScript(schema.getName(), null); } if (doesSummary) ensureCompatibleSummary(field, zName, field.getName(), @@ -118,7 +118,7 @@ public class CreatePositionZCurve extends Processor { ScriptExpression script = inputField.getIndexingScript(); script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); return field; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java index 2e9c23dbf06..487aff81c68 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java @@ -39,20 +39,6 @@ public class DynamicSummaryTransformUtils { return (type.equals(DataType.getArray(DataType.STRING))); } - /** - * Whether a summary field must be populated by the source field with the given type in an indexing script. - */ - public static boolean summaryFieldIsPopulatedBySourceField(DataType sourceFieldType) { - return false; - } - - /** - * Whether a summary field is required as an extra field in the document type. - */ - public static boolean summaryFieldIsRequiredInDocumentType(SummaryField summaryField) { - return summaryFieldIsPopulatedBySourceField(summaryField.getDataType()); - } - public static String getSource(SummaryField summaryField, Schema schema) { // Summary fields with the original supported type is always present in the document type. // However, if the source of that summary field is a single explicit source that exists in the schema we diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index a12183262c4..056c37a9830 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -75,7 +75,7 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript((ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java index 53ebd136e08..a5ba67d6976 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -36,7 +36,7 @@ public class IndexingInputs extends Processor { if (validate) new VerifyInputExpression(schema, field).visit(script); - field.setIndexingScript(script); + field.setIndexingScript(schema.getName(), script); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 071c2878ae8..51fb6b2b065 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -34,33 +34,25 @@ public class IndexingOutputs extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - Set<String> summaryFields = new TreeSet<>(); - findSummaryTo(schema, field, summaryFields, summaryFields); - MyConverter converter = new MyConverter(schema, field, summaryFields, validate); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + findSummaryTo(schema, field); + MyConverter converter = new MyConverter(schema, field, validate); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } - public void findSummaryTo(Schema schema, SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { + public void findSummaryTo(Schema schema, SDField field) { var summaryFields = schema.getSummaryFields(field); - if (summaryFields.isEmpty()) { - fillSummaryToFromField(field, dynamicSummary, staticSummary); - } else { - fillSummaryToFromSearch(schema, field, summaryFields, dynamicSummary, staticSummary); - } + fillSummaryToFromSearch(schema, field, summaryFields); } - private void fillSummaryToFromSearch(Schema schema, SDField field, List<SummaryField> summaryFields, - Set<String> dynamicSummary, Set<String> staticSummary) { + private void fillSummaryToFromSearch(Schema schema, SDField field, List<SummaryField> summaryFields) { for (SummaryField summaryField : summaryFields) { - fillSummaryToFromSummaryField(schema, field, summaryField, dynamicSummary, staticSummary); + fillSummaryToFromSummaryField(schema, field, summaryField); } } - private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField, - Set<String> dynamicSummary, Set<String> staticSummary) { + private void fillSummaryToFromSummaryField(Schema schema, SDField field, SummaryField summaryField) { SummaryTransform summaryTransform = summaryField.getTransform(); - String summaryName = summaryField.getName(); if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { // Avoid writing to summary fields that have more than a single input field, as that is handled by the // summary rewriter in the search core. @@ -68,30 +60,11 @@ public class IndexingOutputs extends Processor { } if (summaryTransform.isDynamic()) { DataType fieldType = field.getDataType(); - if (!DynamicSummaryTransformUtils.summaryFieldIsPopulatedBySourceField(fieldType)) { - if (!DynamicSummaryTransformUtils.isSupportedType(fieldType)) { - warn(schema, field, "Dynamic summaries are only supported for fields of type " + - "string and array<string>, ignoring summary field '" + summaryField.getName() + - "' for sd field '" + field.getName() + "' of type " + - fieldType.getName() + "."); - } - return; - } - dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE && - summaryTransform != SummaryTransform.TOKENS && - summaryTransform != SummaryTransform.ATTRIBUTE_TOKENS) { - staticSummary.add(summaryName); - } - } - - private static void fillSummaryToFromField(SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { - for (SummaryField summaryField : field.getSummaryFields().values()) { - String summaryName = summaryField.getName(); - if (summaryField.getTransform().isDynamic()) { - dynamicSummary.add(summaryName); - } else { - staticSummary.add(summaryName); + if (!DynamicSummaryTransformUtils.isSupportedType(fieldType)) { + warn(schema, field, "Dynamic summaries are only supported for fields of type " + + "string and array<string>, ignoring summary field '" + summaryField.getName() + + "' for sd field '" + field.getName() + "' of type " + + fieldType.getName() + "."); } } } @@ -100,13 +73,11 @@ public class IndexingOutputs extends Processor { final Schema schema; final Field field; - final Set<String> summaryFields; final boolean validate; - MyConverter(Schema schema, Field field, Set<String> summaryFields, boolean validate) { + MyConverter(Schema schema, Field field, boolean validate) { this.schema = schema; this.field = field; - this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; this.validate = validate; } @@ -134,18 +105,7 @@ public class IndexingOutputs extends Processor { } else if (exp instanceof IndexExpression) { ret.add(new IndexExpression(field.getName())); } else if (exp instanceof SummaryExpression) { - for (String fieldName : summaryFields) { - ret.add(new SummaryExpression(fieldName)); - } - /* - * Write to summary field source. AddExtraFieldsToDocument processor adds the "copy" - * summary transform to summary fields without a corresponding explicitly declared - * document field (2023-11-01). Future vespa versions will stop adding document - * fields for those summary fields. - */ - if (!summaryFields.contains(field.getName())) { - ret.add(new SummaryExpression(field.getName())); - } + ret.add(new SummaryExpression(field.getName())); } else { throw new UnsupportedOperationException(exp.getClass().getName()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java index 0d296783cfb..37815ef5396 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java @@ -38,7 +38,7 @@ public class IntegerIndex2Attribute extends Processor { ScriptExpression script = field.getIndexingScript(); Set<String> attributeNames = new HashSet<>(); new MyVisitor(attributeNames).visit(script); - field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyConverter(attributeNames).convert(script)); warn(schema, field, "Changed to attribute because numerical indexes (field has type " + field.getDataType().getName() + ") is not currently supported." + " Index-only settings may fail. Ignore this warning for streaming search."); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index 2ec5c03e04c..c06e0565109 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -46,7 +46,7 @@ public class NGramMatch extends Processor { field.getNormalizing().inferCodepoint(); field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing field.addQueryCommand("ngram " + n); - field.setIndexingScript((ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, n).convert(field.getIndexingScript())); } private static class MyProvider extends TypedTransformProvider { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java index b268a7a9c03..fbb49497837 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java @@ -27,7 +27,7 @@ public class OptimizeIlscript extends Processor { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; - field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)new ExpressionOptimizer().convert(script)); if ( ! field.getIndexingScript().toString().equals(script.toString())) { info(schema, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 2a654991835..c451df6370d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -107,7 +107,7 @@ public class PredicateProcessor extends Processor { script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); ExpressionConverter converter = new PredicateOutputTransformer(schema); - field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); + field.setIndexingScript(schema.getName(), new ScriptExpression((StatementExpression)converter.convert(script))); } private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index c23d87e9eba..52078326aec 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.TestProperties; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Set; /** @@ -28,7 +28,7 @@ public class Processing { public Processing(ModelContext.Properties properties) { this.properties = properties; } private Collection<ProcessorFactory> processors() { - return Arrays.asList( + return List.<ProcessorFactory>of( SearchMustHaveDocument::new, UrlFieldValidator::new, BuiltInFieldSets::new, @@ -99,7 +99,7 @@ public class Processing { /** Processors of rank profiles only (those who tolerate and do something useful when the search field is null) */ private Collection<ProcessorFactory> rankProfileProcessors() { - return Arrays.asList( + return List.of( RankProfileTypeSettingsProcessor::new, ReservedFunctionNames::new, RankingExpressionTypeResolver::new); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index beaff13c613..dd36bbb3b61 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -87,7 +87,7 @@ public abstract class Processor { implementationField.setRankType(RankType.EMPTY); implementationField.setStemming(Stemming.NONE); implementationField.getNormalizing().inferCodepoint(); - implementationField.parseIndexingScript(indexing); + implementationField.parseIndexingScript(schema.getName(), indexing); String indexName = field.getName(); String implementationIndexName = indexName + "_" + suffix; Index implementationIndex = new Index(implementationIndexName); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e6fed35b821..e29f683761f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -48,7 +48,7 @@ public class TextMatch extends Processor { if ( ! visitor.requiresTokenize) continue; ExpressionConverter converter = new MyStringTokenizer(schema, findAnnotatorConfig(schema, field)); - field.setIndexingScript((ScriptExpression)converter.convert(script)); + field.setIndexingScript(schema.getName(), (ScriptExpression)converter.convert(script)); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java index e4e2ac9f5be..9a4c4038953 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java @@ -12,7 +12,6 @@ import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.Arrays; import java.util.List; /** @@ -21,7 +20,7 @@ import java.util.List; public class UriHack extends Processor { private static final List<String> URL_SUFFIX = - Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); + List.of("scheme", "host", "port", "path", "query", "fragment", "hostname"); UriHack(Schema schema, DeployLogger deployLogger, @@ -52,8 +51,7 @@ public class UriHack extends Processor { if (uriField.getDataType() instanceof ArrayDataType) { generatedType = new ArrayDataType(DataType.STRING); } - else if (uriField.getDataType() instanceof WeightedSetDataType) { - WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); + else if (uriField.getDataType() instanceof WeightedSetDataType wdt) { generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java index 111f5384c41..2854824e5b7 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java @@ -5,8 +5,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.schema.document.SDField; import com.yahoo.schema.Schema; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.logging.Level; @@ -16,17 +14,10 @@ import java.util.logging.Level; public class IndexCommandResolver extends MultiFieldResolver { /** Commands which don't have to be harmonized between fields */ - private static List<String> ignoredCommands = new ArrayList<>(); + private static final List<String> ignoredCommands = List.of( "complete-boost", "literal-boost", "highlight"); /** Commands which must be harmonized between fields */ - private static List<String> harmonizedCommands = new ArrayList<>(); - - static { - String[] ignore = { "complete-boost", "literal-boost", "highlight" }; - ignoredCommands.addAll(Arrays.asList(ignore)); - String[] harmonize = { "stemming", "normalizing" }; - harmonizedCommands.addAll(Arrays.asList(harmonize)); - } + private static final List<String> harmonizedCommands = List.of("stemming", "normalizing"); public IndexCommandResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) { super(indexName, fields, schema, logger); @@ -53,8 +44,8 @@ public class IndexCommandResolver extends MultiFieldResolver { ", adding to field " + field.getName()); field.addQueryCommand(command); } else { - deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() + - "\' doesn't contain command \'" + command+"\'"); + deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field '" + field.getName() + + "' doesn't contain command '" + command+"'"); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index f1d3b38e8ff..348a5d575eb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.NetworkPorts; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,7 +27,7 @@ public class HostPorts { private int allocatedPorts = 0; - private PortFinder portFinder = new PortFinder(Collections.emptyList()); + private PortFinder portFinder = new PortFinder(List.of()); private boolean flushed = false; private Optional<NetworkPorts> networkPortsList = Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 1c4b0d31ab2..c876976917b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.time.Clock; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -96,7 +95,7 @@ public class VespaModelFactory implements ModelFactory { } else { this.configModelRegistry = configModelRegistry; } - this.modelImporters = Collections.emptyList(); + this.modelImporters = List.of(); this.additionalValidators = List.of(); this.zone = zone; this.clock = clock; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 78ef6826d26..c19d186df42 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; import java.nio.file.Path; -import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -22,7 +21,7 @@ import java.util.Set; */ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { - private static final Set<Path> UNNECESSARY_BUNDLES = Collections.unmodifiableSet(PlatformBundles.VESPA_SECURITY_BUNDLES); + private static final Set<Path> UNNECESSARY_BUNDLES = Set.copyOf(PlatformBundles.VESPA_SECURITY_BUNDLES); private final ReindexingContext reindexingContext; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 0d85696d503..913ae0e1a00 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -36,7 +36,6 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.nio.file.Path; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -198,7 +197,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC private Map<String, MetricsConsumer> getUserMetricsConsumers() { return getAdmin() .map(admin -> admin.getUserMetrics().getConsumers()) - .orElse(Collections.emptyMap()); + .orElse(Map.of()); } private Optional<String> getSystemName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index aa476b1ae39..0a933e7d5ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -7,7 +7,6 @@ import ai.vespa.metrics.set.Vespa9VespaMetricSet; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.ValuesFetcher; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -71,7 +70,7 @@ public class MetricsConsumer { } public static MetricsConsumer consumer(String id, MetricSet ... metricSets) { - return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets))); + return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), List.of(metricSets))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java new file mode 100644 index 00000000000..fcd587622da --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -0,0 +1,67 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.otel; + +import com.yahoo.cloud.config.OpenTelemetryConfig; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.config.model.producer.AnyConfigProducer; + +import java.util.Optional; + +public class OpenTelemetryCollector extends AbstractService implements OpenTelemetryConfig.Producer { + + private final Zone zone; + + public OpenTelemetryCollector(TreeConfigProducer<?> parent) { + super(parent, "otelcol"); + this.zone = null; + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + } + + public OpenTelemetryCollector(TreeConfigProducer<?> parent, DeployState deployState) { + super(parent, "otelcol"); + this.zone = deployState.zone(); + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + } + + /** + * @return the startup command for the otelcol wrapper + */ + @Override + public Optional<String> getStartupCommand() { + return Optional.of("exec $ROOT/bin/vespa-otelcol-start -c " + getConfigId()); + } + + @Override + public void allocatePorts(int start, PortAllocBridge from) {} + + @Override + public int getPortCount() { + return 0; + } + + @Override + public void getConfig(OpenTelemetryConfig.Builder builder) { + var generator = new OpenTelemetryConfigGenerator(zone); + AnyConfigProducer pp = this; + AnyConfigProducer p = pp.getParent(); + while (p != null && p != pp) { + if (pp instanceof ApplicationConfigProducerRoot root) { + generator.addStatePorts(root.getStatePorts()); + break; + } + pp = p; + p = pp.getParent(); + } + builder.yaml(generator.generate()); + builder.refPaths(generator.referencedPaths()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java new file mode 100644 index 00000000000..36eab6a04b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -0,0 +1,206 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.otel; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.provision.Zone; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * @author olaa + */ +public class OpenTelemetryConfigGenerator { + + private final boolean useTls; + private final String ca_file; + private final String cert_file; + private final String key_file; + private List<StatePortInfo> statePorts = new ArrayList<>(); + + OpenTelemetryConfigGenerator(Zone zone) { + boolean isCd = true; + boolean isPublic = true; + if (zone != null) { + isCd = zone.system().isCd(); + isPublic = zone.system().isPublic(); + this.useTls = true; + } else { + // for manual testing + this.useTls = false; + } + if (isCd) { + if (isPublic) { + this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.external.cd.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.external.cd.tenant.key.pem"; + } else { + this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.vespa.cd.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.vespa.cd.tenant.key.pem"; + } + } else { + if (isPublic) { + this.ca_file = "/opt/vespa/var/vespa/trust-store.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.external.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.external.tenant.key.pem"; + } else { + this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; + this.cert_file = "/var/lib/sia/certs/vespa.vespa.tenant.cert.pem"; + this.key_file = "/var/lib/sia/keys/vespa.vespa.tenant.key.pem"; + } + } + } + + String receiverName(int index) { + return "prometheus_simple/s" + index; + } + + private void addReceivers(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("receivers"); + g.writeStartObject(); + int counter = 0; + for (var statePort : statePorts) { + addReceiver(g, ++counter, statePort); + } + g.writeEndObject(); // receivers + } + private void addReceiver(JsonGenerator g, int index, StatePortInfo statePort) throws java.io.IOException { + g.writeFieldName(receiverName(index)); + g.writeStartObject(); + g.writeStringField("collection_interval", "60s"); + g.writeStringField("endpoint", statePort.hostName() + ":" + statePort.portNumber()); + addUrlInfo(g); + if (useTls) addTls(g); + { + g.writeFieldName("labels"); + g.writeStartObject(); + g.writeStringField("service_type", statePort.serviceType()); + g.writeEndObject(); + } + g.writeEndObject(); + } + private void addTls(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("tls"); + g.writeStartObject(); + g.writeStringField("ca_file", ca_file); + g.writeStringField("cert_file", cert_file); + g.writeBooleanField("insecure_skip_verify", true); + g.writeStringField("key_file", key_file); + g.writeEndObject(); // tls + } + private void addUrlInfo(JsonGenerator g) throws java.io.IOException { + g.writeStringField("metrics_path", "/state/v1/metrics"); + g.writeFieldName("params"); + g.writeStartObject(); + g.writeStringField("format", "prometheus"); + g.writeEndObject(); + } + private void addExporters(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("exporters"); + g.writeStartObject(); + addFileExporter(g); + g.writeEndObject(); // exporters + } + private void addFileExporter(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("file"); + g.writeStartObject(); + g.writeStringField("path", getDefaults().underVespaHome("logs/vespa/otel-test.json")); + { + g.writeFieldName("rotation"); + g.writeStartObject(); + g.writeNumberField("max_megabytes", 10); + g.writeNumberField("max_days", 3); + g.writeNumberField("max_backups", 1); + g.writeEndObject(); // rotation + } + g.writeEndObject(); // file + } + private void addServiceBlock(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("service"); + g.writeStartObject(); + { + g.writeFieldName("telemetry"); + g.writeStartObject(); + { + g.writeFieldName("logs"); + g.writeStartObject(); + g.writeStringField("level", "debug"); + g.writeEndObject(); + } + g.writeEndObject(); + } + { + g.writeFieldName("pipelines"); + g.writeStartObject(); + addMetricsPipelines(g); + g.writeEndObject(); // pipelines + } + g.writeEndObject(); // service + } + private void addMetricsPipelines(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metrics"); + g.writeStartObject(); + { + g.writeFieldName("receivers"); + g.writeStartArray(); + int counter = 0; + for (var statePort : statePorts) { + g.writeString(receiverName(++counter)); + } + g.writeEndArray(); + } + g.writeFieldName("processors"); + g.writeStartArray(); + g.writeEndArray(); + { + g.writeFieldName("exporters"); + g.writeStartArray(); + g.writeString("file"); + g.writeEndArray(); + } + g.writeEndObject(); // metrics + } + + // For now - mostly dummy config + /* + TODO: Create config + 1. polling /state/v1 handler of every service (mostly done) + 2. Processing with mapping/filtering from metric sets + 3. Exporter to correct endpoint (alternatively amended) + */ + public String generate() { + if (statePorts.isEmpty()) { + return ""; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); + g.writeStartObject(); + addReceivers(g); + addExporters(g); + addServiceBlock(g); + g.writeEndObject(); // root + g.close(); + } catch (java.io.IOException e) { + System.err.println("unexpected error: " + e); + return ""; + } + return out.toString(StandardCharsets.UTF_8); + } + + void addStatePorts(List<StatePortInfo> portList) { + this.statePorts = portList; + } + + List<String> referencedPaths() { + return List.of(ca_file, cert_file, key_file); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index 0a23e25e432..4b4cb1530d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import org.w3c.dom.Document; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +34,7 @@ public class BundleValidator extends AbstractBundleValidator { for (Map.Entry<Object,Object> entry : attributes.entrySet()) { mfAttributes.add(entry.getKey().toString()); } - List<String> requiredOSGIHeaders = Arrays.asList( + List<String> requiredOSGIHeaders = List.of( "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java index 9a8c8435790..5e6bd2a4b7f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudClientsValidator.java @@ -31,20 +31,17 @@ public class CloudClientsValidator implements Validator { if (extensions == null) return; // Certificate without any extensions is okay if (extensions.getExtensionOIDs().length == 0) { /* - BouncyCastle 1.77 no longer accepts certificates having an empty sequence of extensions. + BouncyCastle 1.77 and 1.78 did not accept certificates having an empty sequence of extensions. Earlier releases violated the ASN.1 specification as the specification forbids empty extension sequence. See https://github.com/bcgit/bc-java/issues/1479. - - Detect such certificates and issue a warning for now. - Validation will be implicitly enforced once we upgrade BouncyCastle past 1.76. + The restriction was lifted on 1.78.1 although it's a reasonble to warn users still. */ var message = "The certificate's ASN.1 structure contains an empty sequence of extensions, " + "which is a violation of the ASN.1 specification. " + "Please update the application package with a new certificate, " + - "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. " + - "Such certificate will no longer be accepted in near future."; + "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. "; state.getDeployLogger() - .logApplicationPackage(Level.WARNING, errorMessage(clusterName, clientId, message)); + .log(Level.INFO, errorMessage(clusterName, clientId, message)); } } catch (CertificateEncodingException e) { reporter.accept(errorMessage(clusterName, clientId, e.getMessage()), e); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java index e0a43f0988a..ddffe20f725 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidator.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import org.w3c.dom.Document; -import java.util.Arrays; +import java.util.List; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; @@ -18,7 +18,7 @@ public class PublicApiBundleValidator extends AbstractBundleValidator { String nonPublicApiAttribute = mf.getMainAttributes().getValue("X-JDisc-Non-PublicApi-Import-Package"); if (nonPublicApiAttribute == null) return; - var nonPublicApisUsed = Arrays.asList(nonPublicApiAttribute.split(",")); + var nonPublicApisUsed = List.of(nonPublicApiAttribute.split(",")); if (! nonPublicApisUsed.isEmpty()) { log(context.deployState(), Level.WARNING, "Jar file '%s' uses non-public Vespa APIs: %s", filename(jar), nonPublicApisUsed); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 7f02ecacc18..692de1769d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.text.XML; +import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -15,6 +16,8 @@ import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilderBase; import com.yahoo.vespa.model.container.Container; import org.w3c.dom.Element; @@ -38,10 +41,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { super(applicationType, multitenant, configServerSpecs); } + private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { + var otelcol = new OpenTelemetryCollector(parent); + otelcol.setHostResource(hostResource); + otelcol.initService(deployState); + } + @Override protected void doBuildAdmin(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); - admin.setLogserver(parseLogserver(deployState, admin, adminE)); + var logserver = parseLogserver(deployState, admin, adminE); + admin.setLogserver(logserver); + if (deployState.featureFlags().logserverOtelCol()) { + // for manual testing + addOtelcol(admin, deployState, logserver.getHostResource()); + } admin.addConfigservers(configservers); admin.addSlobroks(getSlobroks(deployState, admin, XML.getChild(adminE, "slobroks"))); if ( ! admin.multitenant()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index f722cf375f3..347bb504857 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -12,6 +13,8 @@ import com.yahoo.vespa.model.admin.Logserver; import com.yahoo.vespa.model.admin.LogserverContainer; import com.yahoo.vespa.model.admin.LogserverContainerCluster; import com.yahoo.vespa.model.admin.Slobrok; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; +import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; @@ -113,9 +116,18 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { logServerCluster.addContainer(container); admin.addAndInitializeService(deployState, hostResource, container); admin.setLogserverContainerCluster(logServerCluster); + if (deployState.featureFlags().logserverOtelCol()) + addOtelcol(admin, deployState, hostResource); context.getConfigModelRepoAdder().add(logserverClusterModel); } + + private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { + var otelcol = new OpenTelemetryCollector(parent, deployState); + otelcol.setHostResource(hostResource); + otelcol.initService(deployState); + } + private Collection<HostResource> allocateHosts(HostSystem hostSystem, String clusterId, NodesSpecification nodesSpecification) { return nodesSpecification.provision(hostSystem, ClusterSpec.Type.admin, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java index 046980087ed..bde12bb1d5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java @@ -151,7 +151,7 @@ public class ComponentsBuilder<T extends ChainedComponent<?>> { private Map<String, ComponentType<?>> unmodifiable(Map<String, ComponentType<?>> outerComponentTypeByComponentName) { return (outerComponentTypeByComponentName != null)? Collections.unmodifiableMap(outerComponentTypeByComponentName): - Collections.emptyMap(); + Map.of(); } public Collection<T> getComponentDefinitions() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java index 565cdf3a5f5..b8de223c0cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.model.container.search.searchchain.GenericTarget; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -20,7 +20,7 @@ import java.util.Map; abstract public class DomGenericTargetBuilder<T extends GenericTarget> extends DomChainBuilderBase<Searcher<?>, T> { DomGenericTargetBuilder(Map<String, ComponentsBuilder.ComponentType<?>> outerSearcherTypeByComponentName) { - super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + super(List.of(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), outerSearcherTypeByComponentName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java index 2eb1127d75b..4ada9e99e5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChain; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -21,7 +21,7 @@ import java.util.Map; public class DomSearchChainBuilder extends DomChainBuilderBase<Searcher<?>, SearchChain> { public DomSearchChainBuilder(Map<String, ComponentsBuilder.ComponentType<?>> outerSearcherTypeByComponentName) { - super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + super(List.of(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), outerSearcherTypeByComponentName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java index e05d762d7bc..b48df948358 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.container.search.searchchain.Searcher; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -23,7 +22,7 @@ import java.util.Map; public class DomSearchChainsBuilder extends DomChainsBuilder<Searcher<?>, SearchChain, SearchChains> { public DomSearchChainsBuilder() { - super(Arrays.asList(ComponentType.searcher, ComponentType.federation)); + super(List.of(ComponentType.searcher, ComponentType.federation)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 62979404025..5be1690f0dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -31,6 +31,7 @@ public class ContainerModelEvaluation implements public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String LLAMA_BUNDLE_NAME = "container-llama.jar"; public final static String ONNX_RUNTIME_CLASS = "ai.vespa.modelintegration.evaluator.OnnxRuntime"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index baa9c4b9a48..468cf8dd961 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -14,6 +14,7 @@ import java.util.stream.Stream; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.LINGUISTICS_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.LLAMA_BUNDLE_NAME; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; /** @@ -37,6 +38,7 @@ public class PlatformBundles { public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); public static final String SEARCH_AND_DOCPROC_BUNDLE = BundleInstantiationSpecification.CONTAINER_SEARCH_AND_DOCPROC; + public static final String MODEL_INTEGRATION_BUNDLE = BundleInstantiationSpecification.MODEL_INTEGRATION; // Bundles that must be loaded for all container types. public static final Set<Path> COMMON_VESPA_BUNDLES = toBundlePaths( @@ -63,13 +65,14 @@ public class PlatformBundles { "lucene-linguistics", EVALUATION_BUNDLE_NAME, INTEGRATION_BUNDLE_NAME, - ONNXRUNTIME_BUNDLE_NAME + ONNXRUNTIME_BUNDLE_NAME, + LLAMA_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { return Stream.of(bundleNames) .map(PlatformBundles::absoluteBundlePath) - .collect(Collectors.toSet()); + .collect(Collectors.toUnmodifiableSet()); } public static Path absoluteBundlePath(String fileName) { @@ -86,6 +89,10 @@ public class PlatformBundles { return searchAndDocprocComponents.contains(className); } + public static boolean isModelIntegrationClass(String className) { + return modelIntegrationComponents.contains(className); + } + // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. private static final Set<String> searchAndDocprocComponents = Set.of( com.yahoo.docproc.AbstractConcreteDocumentFactory.class.getName(), @@ -147,8 +154,13 @@ public class PlatformBundles { com.yahoo.vespa.streamingvisitors.MetricsSearcher.class.getName(), com.yahoo.vespa.streamingvisitors.StreamingBackend.class.getName(), ai.vespa.search.llm.LLMSearcher.class.getName(), - ai.vespa.search.llm.RAGSearcher.class.getName(), - ai.vespa.llm.clients.OpenAI.class.getName() + ai.vespa.search.llm.RAGSearcher.class.getName() + ); + + // This is a hack to allow users to declare components from the model-integration bundle without naming the bundle. + private static final Set<String> modelIntegrationComponents = Set.of( + ai.vespa.llm.clients.OpenAI.class.getName(), + ai.vespa.llm.clients.LocalLLM.class.getName() ); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index ed626f98aa0..dafac2c1af9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container.component; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -24,9 +23,9 @@ public class DiscBindingsConfigGenerator { public static <T extends Handler> Map<String, Handlers.Builder> generate(T handler) { if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) - return Collections.emptyMap(); + return Map.of(); - return Collections.singletonMap(handler.model.getComponentId().stringValue(), + return Map.of(handler.model.getComponentId().stringValue(), new Handlers.Builder() .serverBindings(toStrings(handler.getServerBindings())) .clientBindings(toStrings(handler.getClientBindings()))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 099255975b6..0d3d8462376 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.model.container.ContainerThreadpool; import org.w3c.dom.Element; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; @@ -51,7 +50,7 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { } public void addServerBindings(BindingPattern... bindings) { - serverBindings.addAll(Arrays.asList(bindings)); + serverBindings.addAll(List.of(bindings)); } public void addServerBindings(Collection<BindingPattern> bps) { serverBindings.addAll(bps); } @@ -61,7 +60,7 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { } public void addClientBindings(BindingPattern... bindings) { - clientBindings.addAll(Arrays.asList(bindings)); + clientBindings.addAll(List.of(bindings)); } public final Collection<BindingPattern> getServerBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java index 7d6285d00c1..3a8e52c92a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; import java.net.URI; +import java.net.URISyntaxException; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -51,11 +52,22 @@ class Model { static Model fromXml(DeployState ds, Element model, Set<String> requiredTags) { var modelId = XmlHelper.getOptionalAttribute(model, "model-id").orElse(null); - var url = XmlHelper.getOptionalAttribute(model, "url").map(URI::create).orElse(null); + var url = XmlHelper.getOptionalAttribute(model, "url").map(Model::parseUrl).orElse(null); var path = XmlHelper.getOptionalAttribute(model, "path").map(Path::fromString).orElse(null); return new Model(ds, model.getTagName(), modelId, url, path, requiredTags); } + private static URI parseUrl(String url) { + try { + var uri = new URI(url); + if ( ! uri.isAbsolute()) + throw new IllegalArgumentException("Invalid url '" + url + "': url has no 'scheme' component"); + return uri; + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid url '" + url + "':" + e.getMessage()); + } + } + /** Return tokenizer model from XML if specified, alternatively use model id for ONNX model with suffix '-vocab' appended */ static Model fromXmlOrImplicitlyFromOnnxModel( DeployState ds, Element parent, Model onnxModel, String paramName, Set<String> requiredTags) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 8c1b868465a..19c45bba2ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -48,7 +48,7 @@ public class AccessControl { private final String domain; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); - private Collection<Handler> handlers = Collections.emptyList(); + private Collection<Handler> handlers = List.of(); public Builder(String domain) { this.domain = domain; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java index bc234546625..82a8a01871e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.model.container.http.HttpFilterChain; import org.w3c.dom.Element; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.Map; import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; @@ -21,7 +21,7 @@ import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.Com */ public class FilterChainBuilder extends DomChainBuilderBase<Filter, HttpFilterChain> { - private static final Collection<ComponentType<Filter>> allowedComponentTypes = Collections.singleton(ComponentType.filter); + private static final Collection<ComponentType<Filter>> allowedComponentTypes = List.of(ComponentType.filter); public FilterChainBuilder(Map<String, ComponentType<?>> outerFilterTypeByComponentName) { super(allowedComponentTypes, outerFilterTypeByComponentName); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java index 878859a01bf..f1cdbfb3b04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.container.http.HttpFilterChain; import org.w3c.dom.Element; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -24,7 +23,7 @@ import java.util.Map; */ public class FilterChainsBuilder extends DomChainsBuilder<Filter, HttpFilterChain, FilterChains> { private static final Collection<ComponentType<Filter>> allowedComponentTypes = - Collections.singleton(ComponentType.filter); + List.of(ComponentType.filter); private static final Map<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends HttpFilterChain>>> chainType2BuilderClass = Map.of( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java index 29e587c6453..f589884cccd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java @@ -4,9 +4,8 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.search.searchchain.model.federation.FederationOptions; -import com.yahoo.config.model.producer.TreeConfigProducer; -import java.util.Arrays; +import java.util.List; /** @@ -51,7 +50,7 @@ public class Source extends GenericTarget { @Override public ChainSpecification getChainSpecification() { return super.getChainSpecification().addInherits( - Arrays.asList(getParentProvider().getComponentId().toSpecification())); + List.of(getParentProvider().getComponentId().toSpecification())); } public ComponentId getGlobalComponentId() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 8a15e61495b..1323506eaeb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.vespa.model.container.PlatformBundles; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; @@ -27,21 +26,20 @@ public class BundleInstantiationSpecificationBuilder { BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); validate(instSpec); - return bundle == null ? setBundleForSearchAndDocprocComponents(instSpec) : instSpec; + return bundle == null ? setBundleForComponent(instSpec) : instSpec; } - private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { + private static BundleInstantiationSpecification setBundleForComponent(BundleInstantiationSpecification spec) { if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) return spec.inBundle(PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + else if (PlatformBundles.isModelIntegrationClass(spec.getClassName())) + return spec.inBundle(PlatformBundles.MODEL_INTEGRATION_BUNDLE); else return spec; } - private static void validate(BundleInstantiationSpecification instSpec) { - List<String> forbiddenClasses = Arrays.asList( - SearchHandler.HANDLER_CLASSNAME, - PROCESSING_HANDLER_CLASS); + List<String> forbiddenClasses = List.of(SearchHandler.HANDLER_CLASSNAME, PROCESSING_HANDLER_CLASS); for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { @@ -50,7 +48,7 @@ public class BundleInstantiationSpecificationBuilder { } } - //null if missing + // null if missing private static ComponentSpecification getComponentSpecification(Element spec, String attributeName) { return (spec.hasAttribute(attributeName)) ? new ComponentSpecification(spec.getAttribute(attributeName)) : diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 021753eb3f3..56e2a21e38b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -114,7 +114,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -507,6 +506,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { boolean atLeastOneClientWithCertificate = clients.stream().anyMatch(client -> !client.certificates().isEmpty()); if (!atLeastOneClientWithCertificate) throw new IllegalArgumentException("At least one client must require a certificate"); + + List<String> duplicates = clients.stream().collect(Collectors.groupingBy(Client::id)) + .entrySet().stream().filter(entry -> entry.getValue().size() > 1) + .map(Map.Entry::getKey).sorted().toList(); + if (! duplicates.isEmpty()) { + throw new IllegalArgumentException("Duplicate client ids: " + duplicates); + } } List<X509Certificate> operatorAndTesterCertificates = deployState.getProperties().operatorCertificates(); @@ -892,7 +898,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addStandaloneNode(ApplicationContainerCluster cluster, DeployState deployState) { ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), deployState); - cluster.addContainers(Collections.singleton(container)); + cluster.addContainers(List.of(container)); } private static String buildJvmGCOptions(ConfigModelContext context, String jvmGCOptions) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index f9993b770e5..867ac86f8d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -26,6 +26,7 @@ public class ModelIdResolver { public static final String ONNX_MODEL = "onnx-model"; public static final String BERT_VOCAB = "bert-vocabulary"; public static final String SIGNIFICANCE_MODEL = "significance-model"; + public static final String GGUF_MODEL = "gguf-model"; private static Map<String, ProvidedModel> setupProvidedModels() { var m = new HashMap<String, ProvidedModel>(); @@ -60,6 +61,9 @@ public class ModelIdResolver { register(m, "e5-large-v2", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/model.onnx", Set.of(ONNX_MODEL)); register(m, "e5-large-v2-vocab", "https://data.vespa.oath.cloud/onnx_models/e5-large-v2/tokenizer.json", Set.of(HF_TOKENIZER)); + + register(m, "mistral-7b", "https://data.vespa.oath.cloud/gguf_models/mistral-7b-instruct-v0.1.Q6_K.gguf", Set.of(GGUF_MODEL)); + register(m, "mistral-7b-q8", "https://data.vespa.oath.cloud/gguf_models/mistral-7b-instruct-v0.1.Q8_0.gguf", Set.of(GGUF_MODEL)); return Map.copyOf(m); } @@ -124,7 +128,7 @@ public class ModelIdResolver { throw new IllegalArgumentException("Unknown model id '" + modelId + "' on '" + valueName + "'. Available models are [" + providedModels.keySet().stream().sorted().collect(Collectors.joining(", ")) + "]"); var providedModel = providedModels.get(modelId); - if (!providedModel.tags().containsAll(requiredTags)) { + if ( ! providedModel.tags().containsAll(requiredTags)) { throw new IllegalArgumentException( "Model '%s' on '%s' has tags %s but are missing required tags %s" .formatted(modelId, valueName, providedModel.tags(), requiredTags)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 9d15eddbef3..6d482a6d4a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -35,7 +35,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -195,7 +194,7 @@ public class Content extends ConfigModel { public static class Builder extends ConfigModelBuilder<Content> { - public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content")); + public static final List<ConfigModelId> configModelIds = List.of(ConfigModelId.fromName("content")); public Builder() { super(Content.class); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 5ec23b53109..ba52664a660 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -37,6 +37,14 @@ public abstract class ContentNode extends AbstractService rpc_num_targets = featureFlags.rpcNumTargets(); rpc_events_before_wakeup = featureFlags.rpcEventsBeforeWakeup(); + // <node>-level distribution key range validation is initially done through the XML schema, + // but we also check it here in the case of programmatic content node instantiations. + // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal + // sentinel value that must never be used by actual nodes. + if (distributionKey < 0 || distributionKey >= 65535) { + throw new IllegalArgumentException("Distribution key %d is outside valid range [0, 65534]".formatted(distributionKey)); + } + initialize(); setProp("clustertype", "content"); setProp("clustername", clusterName); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java index 53547a1a15b..6dd7278fde8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java @@ -3,9 +3,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -13,9 +10,8 @@ import java.util.stream.Collectors; public class ReservedDocumentTypeNameValidator { - public static final List<String> ORDERED_RESERVED_NAMES = Collections.unmodifiableList( - Arrays.asList("and", "false", "id", "not", "null", "or", "true")); - public static final Set<String> RESERVED_NAMES = Collections.unmodifiableSet(new HashSet<>(ORDERED_RESERVED_NAMES)); + public static final List<String> ORDERED_RESERVED_NAMES = List.of("and", "false", "id", "not", "null", "or", "true"); + public static final Set<String> RESERVED_NAMES = Set.copyOf(ORDERED_RESERVED_NAMES); public void validate(Map<String, NewDocumentType> documentDefinitions) { List<String> conflictingNames = documentDefinitions.keySet().stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index a6f53777c51..a1b9ed67aa7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.model.content.engines.PersistenceEngine; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -349,7 +348,7 @@ public class StorageGroup { owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), context) : - Collections.emptyMap(); + Map.of(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); if (hostGroups.size() > 1) { @@ -475,7 +474,7 @@ public class StorageGroup { } private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) { - if (groupOrNodesElement.isEmpty()) return Collections.emptyList(); + if (groupOrNodesElement.isEmpty()) return List.of(); List<XmlNodeBuilder> nodes = new ArrayList<>(); for (ModelElement n : groupOrNodesElement.get().subElements("node")) nodes.add(new XmlNodeBuilder(clusterElement, n)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java index c9b6537434b..07dafda164d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilder.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -20,19 +19,19 @@ public class GlobalDistributionBuilder { private final Map<String, NewDocumentType> documentDefinitions; public GlobalDistributionBuilder(Map<String, NewDocumentType> documentDefinitions) { - this.documentDefinitions = Collections.unmodifiableMap(documentDefinitions); + this.documentDefinitions = Map.copyOf(documentDefinitions); } public Set<NewDocumentType> build(ModelElement documentsElement) { if (documentsElement == null || documentsElement.subElements("document").isEmpty()) - return Collections.emptySet(); + return Set.of(); return documentsElement.subElements("document") .stream() .filter(GlobalDistributionBuilder::isGloballyDistributed) .map(GlobalDistributionBuilder::getDocumentName) .map(this::getDocumentType) - .collect(Collectors.toCollection(() -> new LinkedHashSet<>())); + .collect(Collectors.toCollection(LinkedHashSet::new)); } private static boolean isGloballyDistributed(ModelElement e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 6fd917c393d..ae84978793a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -47,6 +47,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final boolean useAsyncMessageHandlingOnSchedule; + private final int persistenceThreadMaxFeedOpBatchSize; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -62,6 +63,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); + this.persistenceThreadMaxFeedOpBatchSize = featureFlags.persistenceThreadMaxFeedOpBatchSize(); } @@ -75,6 +77,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); builder.async_operation_throttler(throttleBuilder); + builder.max_feed_op_batch_size(persistenceThreadMaxFeedOpBatchSize); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 621377e0606..78ed10a583e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -32,7 +32,6 @@ import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -177,7 +176,7 @@ public class ConvertedModel { if (arguments.output().isEmpty()) { List<Map.Entry<String, ExpressionFunction>> entriesWithTheRightPrefix = expressions.entrySet().stream().filter(entry -> entry.getKey().startsWith(arguments.signature().get() + ".")).toList(); - if (entriesWithTheRightPrefix.size() < 1) + if (entriesWithTheRightPrefix.isEmpty()) throw new IllegalArgumentException("No expressions named '" + arguments.signature().get() + missingExpressionMessageSuffix()); if (entriesWithTheRightPrefix.size() > 1) @@ -191,7 +190,7 @@ public class ConvertedModel { private String missingExpressionMessageSuffix() { return "' in model '" + modelDescription + "'. " + - "Available expressions: " + expressions.keySet().stream().collect(Collectors.joining(", ")); + "Available expressions: " + String.join(", ", expressions.keySet()); } // ----------------------- Static model conversion/storage below here @@ -425,8 +424,7 @@ public class ConvertedModel { } private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedMlModel model) { - if (node instanceof ReferenceNode) { - ReferenceNode referenceNode = (ReferenceNode)node; + if (node instanceof ReferenceNode referenceNode) { if (referenceNode.getOutput() == null) { // function references cannot specify outputs if (names.add(referenceNode.getName())) { if (model.functions().containsKey(referenceNode.getName())) { @@ -485,7 +483,7 @@ public class ConvertedModel { List<Pair<String, ExpressionFunction>> readExpressions() { List<Pair<String, ExpressionFunction>> expressions = new ArrayList<>(); ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath()); - if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyList(); + if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return List.of(); for (ApplicationFile expressionFile : expressionPath.listFiles()) { try (BufferedReader reader = new BufferedReader(expressionFile.createReader())) { String name = expressionFile.getPath().getName(); @@ -525,7 +523,7 @@ public class ConvertedModel { List<Pair<String, RankingExpression>> readFunctions() { try { ApplicationFile file = application.getFile(modelFiles.functionsPath()); - if ( ! file.exists()) return Collections.emptyList(); + if ( ! file.exists()) return List.of(); List<Pair<String, RankingExpression>> functions = new ArrayList<>(); try (BufferedReader reader = new BufferedReader(file.createReader())) { @@ -597,7 +595,7 @@ public class ConvertedModel { private List<Pair<String, Tensor>> readSmallConstants() { try { ApplicationFile file = application.getFile(modelFiles.smallConstantsPath()); - if ( ! file.exists()) return Collections.emptyList(); + if ( ! file.exists()) return List.of(); List<Pair<String, Tensor>> constants = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 502924ed31a..f47a48b09d7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -17,7 +17,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -51,8 +50,8 @@ public class OnnxModelInfo { Map<String, OnnxTypeInfo> outputs, Set<String> initializers, String defaultOutput) { this.app = app; this.modelPath = path; - this.inputs = Collections.unmodifiableMap(inputs); - this.outputs = Collections.unmodifiableMap(outputs); + this.inputs = Map.copyOf(inputs); + this.outputs = Map.copyOf(outputs); this.defaultOutput = defaultOutput; this.initializers = Set.copyOf(initializers); } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 520f41609b2..0601741bb16 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -207,7 +207,7 @@ Documents = element documents { ContentNode = element node { GenericConfig* & service.attlist & - attribute distribution-key { xsd:nonNegativeInteger } & + attribute distribution-key { xsd:nonNegativeInteger { maxInclusive = "65534" } } & attribute capacity { xsd:double { minExclusive = "0.0" } }? & attribute mmap-core-limit { xsd:nonNegativeInteger }? & attribute core-on-oom { xsd:boolean }? & diff --git a/config-model/src/test/cfg/application/embed/services.xml b/config-model/src/test/cfg/application/embed/services.xml index e92679e3c96..089adbd7517 100644 --- a/config-model/src/test/cfg/application/embed/services.xml +++ b/config-model/src/test/cfg/application/embed/services.xml @@ -40,7 +40,7 @@ <component id="bert-embedder" type="bert-embedder"> <!-- model specifics --> - <transformer-model model-id="minilm-l6-v2" url="application-url"/> + <transformer-model model-id="minilm-l6-v2" url="https://my/url/model.onnx"/> <tokenizer-vocab path="files/vocab.txt"/> <max-tokens>512</max-tokens> <transformer-input-ids>my_input_ids</transformer-input-ids> diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index f023328380c..2b8a6fc344d 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -9,10 +9,6 @@ indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "type Reference<parent>" indexinfo[].command[].indexname "parent_ref" indexinfo[].command[].command "word" -indexinfo[].command[].indexname "documentid" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "documentid" -indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_array.name" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 14a7a62f4bb..0cdf921de25 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -17,7 +17,7 @@ ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attrib ilscript[].content[] "clear_state | guard { input a | summary a | attribute a; }" ilscript[].content[] "clear_state | guard { input adynamic | summary adynamic | attribute adynamic; }" ilscript[].content[] "clear_state | guard { input abolded | summary abolded | attribute abolded; }" -ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }" +ilscript[].content[] "clear_state | guard { input b | summary b; }" ilscript[].content[] "clear_state | guard { input c | summary c | attribute c; }" ilscript[].content[] "clear_state | guard { input d | summary d; }" ilscript[].content[] "clear_state | guard { input e | summary e; }" diff --git a/config-model/src/test/derived/multiplesummaries/index-info.cfg b/config-model/src/test/derived/multiplesummaries/index-info.cfg index 65ffd71d1ca..58759be9398 100644 --- a/config-model/src/test/derived/multiplesummaries/index-info.cfg +++ b/config-model/src/test/derived/multiplesummaries/index-info.cfg @@ -83,16 +83,6 @@ indexinfo[].command[].indexname "mytags" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mytags" indexinfo[].command[].command "type Array<string>" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "multivalue" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "alltags" -indexinfo[].command[].command "type Array<string>" -indexinfo[].command[].indexname "anotherb" -indexinfo[].command[].command "string" -indexinfo[].command[].indexname "anotherb" -indexinfo[].command[].command "type string" indexinfo[].command[].indexname "loc_pos.x" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "loc_pos.x" diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index f916cc26c36..0b503b04926 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -143,10 +143,4 @@ doctype[].structtype[].field[].type 10017 doctype[].structtype[].field[].name "g" doctype[].structtype[].field[].internalid 1091070635 doctype[].structtype[].field[].type 10012 -doctype[].structtype[].field[].name "snippet2" -doctype[].structtype[].field[].internalid 1812076817 -doctype[].structtype[].field[].type 10012 -doctype[].structtype[].field[].name "anothersummaryfield" -doctype[].structtype[].field[].internalid 1811005492 -doctype[].structtype[].field[].type 10012 diff --git a/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java b/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java index cb232c4f812..900799b86a1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/MapConfigModelRegistryTest.java @@ -7,7 +7,6 @@ import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import java.util.Collection; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -47,7 +46,7 @@ public class MapConfigModelRegistryTest { } @Override - public List<ConfigModelId> handlesElements() { return Collections.singletonList(ConfigModelId.fromName("modelB")); } + public List<ConfigModelId> handlesElements() { return List.of(ConfigModelId.fromName("modelB")); } @Override public void doBuild(ModelB model, Element spec, ConfigModelContext modelContext) { } } @@ -57,7 +56,7 @@ public class MapConfigModelRegistryTest { super(ModelB.class); } @Override - public List<ConfigModelId> handlesElements() { return Collections.singletonList(ConfigModelId.fromName("modelB")); } + public List<ConfigModelId> handlesElements() { return List.of(ConfigModelId.fromName("modelB")); } @Override public void doBuild(ModelB model, Element spec, ConfigModelContext modelContext) { } } @@ -73,7 +72,7 @@ public class MapConfigModelRegistryTest { super(ModelA.class); } @Override - public List<ConfigModelId> handlesElements() { return Collections.singletonList(ConfigModelId.fromName("modelA")); } + public List<ConfigModelId> handlesElements() { return List.of(ConfigModelId.fromName("modelA")); } @Override public void doBuild(ModelA model, Element spec, ConfigModelContext modelContext) { } diff --git a/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java b/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java index 327d48f9276..4c6a5d0e74b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java +++ b/config-model/src/test/java/com/yahoo/config/model/graph/GraphMock.java @@ -8,7 +8,6 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -20,7 +19,7 @@ public class GraphMock { public static class BA extends ConfigModelBuilder<A> { public BA() { super(A.class); } - @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public List<ConfigModelId> handlesElements() { return List.of(); } @Override public void doBuild(A model, Element spec, ConfigModelContext modelContext) { } } public static class A extends ConfigModel { @@ -29,7 +28,7 @@ public class GraphMock { public static class BB extends ConfigModelBuilder<B> { public BB() { super(B.class); } - @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public List<ConfigModelId> handlesElements() { return List.of(); } @Override public void doBuild(B model, Element spec, ConfigModelContext modelContext) { } } public static class B extends ConfigModel { @@ -40,7 +39,7 @@ public class GraphMock { public static class BC extends ConfigModelBuilder<C> { public BC() { super(C.class); } - @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public List<ConfigModelId> handlesElements() { return List.of(); } @Override public void doBuild(C model, Element spec, ConfigModelContext modelContext) { } } public static class C extends ConfigModel { @@ -51,7 +50,7 @@ public class GraphMock { public static class BD extends ConfigModelBuilder<D> { public BD() { super(D.class); } - @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public List<ConfigModelId> handlesElements() { return List.of(); } @Override public void doBuild(D model, Element spec, ConfigModelContext modelContext) { } } public static class D extends ConfigModel { @@ -60,7 +59,7 @@ public class GraphMock { public static class BE extends ConfigModelBuilder<E> { public BE() { super(E.class); } - @Override public List<ConfigModelId> handlesElements() { return Arrays.asList(); } + @Override public List<ConfigModelId> handlesElements() { return List.of(); } @Override public void doBuild(E model, Element spec, ConfigModelContext modelContext) { } } public static class E extends ConfigModel { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java index 51574432e6d..02ca9effffb 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test; import java.io.StringReader; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -22,27 +21,29 @@ import static org.junit.jupiter.api.Assertions.*; */ public class HostsXmlProvisionerTest { - private static final String oneHost = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + - "<hosts>\n" + - " <host name=\"test1.yahoo.com\">\n" + - " <alias>node1</alias>\n" + - " <alias>node2</alias>\n" + - " </host>\n" + - "</hosts>"; - - private static final String threeHosts = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + - "<hosts>\n" + - " <host name=\"test1.yahoo.com\">\n" + - " <alias>node1</alias>\n" + - " </host>\n" + - " <host name=\"test2.yahoo.com\">\n" + - " <alias>node2</alias>\n" + - " <alias>node3</alias>\n" + - " </host>\n" + - " <host name=\"test3.yahoo.com\">\n" + - " <alias>node4</alias>\n" + - " </host>\n" + - "</hosts>"; + private static final String oneHost = """ + <?xml version="1.0" encoding="utf-8"?> + <hosts> + <host name="test1.yahoo.com"> + <alias>node1</alias> + <alias>node2</alias> + </host> + </hosts>"""; + + private static final String threeHosts = """ + <?xml version="1.0" encoding="utf-8"?> + <hosts> + <host name="test1.yahoo.com"> + <alias>node1</alias> + </host> + <host name="test2.yahoo.com"> + <alias>node2</alias> + <alias>node3</alias> + </host> + <host name="test3.yahoo.com"> + <alias>node4</alias> + </host> + </hosts>"""; @Test void require_basic_works() { @@ -57,7 +58,7 @@ public class HostsXmlProvisionerTest { assertTrue(map.keySet().containsAll(aliases)); // 5 services, 3 host aliases, mapping to 2 host. - aliases = createAliases(Collections.singletonList("node3")); + aliases = createAliases(List.of("node3")); map = allocate(hostProvisioner, aliases); assertCorrectNumberOfHosts(map, 2); @@ -65,7 +66,7 @@ public class HostsXmlProvisionerTest { assertTrue(map.keySet().containsAll(aliases)); // 5 services, 3 host aliases, mapping to 3 host. - aliases = createAliases(Collections.singletonList("node4")); + aliases = createAliases(List.of("node4")); map = allocate(hostProvisioner, aliases); assertEquals(3, map.size()); assertCorrectNumberOfHosts(map, 3); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java index e5dca4e5adf..b4f586e3f2f 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/SingleNodeProvisionerTest.java @@ -12,7 +12,6 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -40,7 +39,7 @@ public class SingleNodeProvisionerTest { assertTrue(map.keySet().containsAll(aliases)); // 5 services, 3 host aliases, mapping to 2 host. - aliases = createAliases(Collections.singletonList("node3")); + aliases = createAliases(List.of("node3")); map = allocate(hostProvisioner, aliases); assertCorrectNumberOfHost(map, 1); @@ -48,7 +47,7 @@ public class SingleNodeProvisionerTest { assertTrue(map.keySet().containsAll(aliases)); // 5 services, 3 host aliases, mapping to 3 host. - aliases = createAliases(Collections.singletonList("node4")); + aliases = createAliases(List.of("node4")); map = allocate(hostProvisioner, aliases); assertThat(map.size(), is(3)); assertCorrectNumberOfHost(map, 1); diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java index 69e7e14b3be..9ae24d6bfd4 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java @@ -8,8 +8,8 @@ import com.yahoo.schema.document.SDField; */ public class AttributeUtils { - public static void addAttributeAspect(SDField field) { - field.parseIndexingScript("{ attribute }"); + public static void addAttributeAspect(String schemaName, SDField field) { + field.parseIndexingScript(schemaName, "{ attribute }"); } } diff --git a/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java index defc7e59080..88761fc5d3f 100644 --- a/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java @@ -9,7 +9,6 @@ import com.yahoo.schema.document.TemporarySDField; import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -142,14 +141,13 @@ public class DocumentGraphValidatorTest { Schema campaignSchema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); campaignSchema.addDocument(document); - document.setDocumentReferences(new DocumentReferences(Collections.emptyMap())); + document.setDocumentReferences(new DocumentReferences(Map.of())); Arrays.stream(parents) .map(Schema::getDocument) .forEach(document::inherit); return campaignSchema; } - @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { SDDocumentType fromDocument = from.getDocument(); SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); diff --git a/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java index 76006ad28d7..99946406344 100644 --- a/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java @@ -8,10 +8,9 @@ import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Map; -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.*; /** @@ -34,13 +33,13 @@ public class DocumentReferenceResolverTest { SDDocumentType fooDocument = new SDDocumentType("foo", fooSchema); SDField fooRefToBarField = new SDField (fooDocument, "bar_ref", new NewDocumentReferenceDataType(barDocument.getDocumentType())); - AttributeUtils.addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooSchema.getName(), fooRefToBarField); SDField irrelevantField = new SDField(fooDocument, "irrelevant_stuff", DataType.INT); fooDocument.addField(fooRefToBarField); fooDocument.addField(irrelevantField); fooSchema.addDocument(fooDocument); - DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); + DocumentReferenceResolver resolver = new DocumentReferenceResolver(List.of(fooSchema, barSchema)); resolver.resolveReferences(fooDocument); assertTrue(fooDocument.getDocumentReferences().isPresent()); @@ -50,7 +49,6 @@ public class DocumentReferenceResolverTest { assertSame(fooRefToBarField, fooReferenceMap.get("bar_ref").referenceField()); } - @SuppressWarnings("deprecation") @Test void throws_user_friendly_exception_if_referenced_document_does_not_exist() { Throwable exception = assertThrows(IllegalArgumentException.class, () -> { @@ -60,11 +58,11 @@ public class DocumentReferenceResolverTest { SDField fooRefToBarField = new SDField( fooDocument, "bar_ref", NewDocumentReferenceDataType.forDocumentName("bar")); - AttributeUtils.addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooSchema.getName(), fooRefToBarField); fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); - DocumentReferenceResolver resolver = new DocumentReferenceResolver(singletonList(fooSchema)); + DocumentReferenceResolver resolver = new DocumentReferenceResolver(List.of(fooSchema)); resolver.resolveReferences(fooDocument); }); assertTrue(exception.getMessage().contains("Invalid document reference 'bar_ref': Could not find document type 'bar'")); @@ -86,7 +84,7 @@ public class DocumentReferenceResolverTest { fooDocument.addField(fooRefToBarField); fooSchema.addDocument(fooDocument); - DocumentReferenceResolver resolver = new DocumentReferenceResolver(asList(fooSchema, barSchema)); + DocumentReferenceResolver resolver = new DocumentReferenceResolver(List.of(fooSchema, barSchema)); resolver.resolveReferences(fooDocument); }); assertTrue(exception.getMessage().contains("The field 'bar_ref' is an invalid document reference. The field must be an attribute.")); diff --git a/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java index 8c0b8c32d81..b96a43e9b53 100644 --- a/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java @@ -22,7 +22,7 @@ public class ImportedFieldsEnumeratorTest { Schema parentSchema = new Schema(PARENT, MockApplicationPackage.createEmpty()); SDDocumentType parentDocument = new SDDocumentType(PARENT, parentSchema); var parentField = new SDField(parentDocument, "their_field", DataType.INT); - AttributeUtils.addAttributeAspect(parentField); + AttributeUtils.addAttributeAspect(parentSchema.getName(), parentField); parentDocument.addField(parentField); parentSchema.addDocument(parentDocument); diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index a11e743b4d2..6252fe62a1d 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -40,7 +40,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { SDDocumentType document = schema.getDocument(); assertEquals("simple", document.getName()); - assertEquals(20, document.getFieldCount()); + assertEquals(19, document.getFieldCount()); SDField field; Attribute attribute; diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 6442edd547d..8ffbab84fd7 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -359,7 +359,7 @@ public class SummaryTestCase { ApplicationBuilder.createFromStrings(logger, sd); if (explicit) { assertEquals(1, logger.entries.size()); - assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("For schema 'test', field 'foo', summary 'bar':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); @@ -392,7 +392,7 @@ public class SummaryTestCase { ApplicationBuilder.createFromStrings(logger, sd); if (explicit) { assertEquals(1, logger.entries.size()); - assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals(Level.WARNING, logger.entries.get(0).level); assertEquals("For schema 'test', document-summary 'bar', summary field 'foo':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java index 0cfb9474365..f21999df94c 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java @@ -14,6 +14,7 @@ import java.util.Iterator; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Tests attribute deriving @@ -126,4 +127,38 @@ public class AttributeListTestCase extends AbstractSchemaTestCase { assertFalse(attributes.hasNext()); } + @Test + void bad_map_attribute() throws ParseException { + run_bad_struct_or_map_attribute("map<string,string>"); + } + + @Test + void bad_array_of_struct_attribute() throws ParseException { + run_bad_struct_or_map_attribute("array<s>"); + } + + private void run_bad_struct_or_map_attribute(String type) throws ParseException { + run_bad_struct_or_map_attribute(type, false, true); + run_bad_struct_or_map_attribute(type, true, false); + run_bad_struct_or_map_attribute(type, true, true); + } + + private void run_bad_struct_or_map_attribute(String type, boolean fs, boolean ilscript) throws ParseException { + var exception = assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString( + joinLines("search test {", + " document test {", + " struct s {", + " field a type string { }", + " }", + " field metadata type " + type + " {", + " indexing: summary" + (ilscript ? "| attribute" : ""), + " " + (fs ? "attribute: fast-search" : ""), + " }", + " }", + "}")).getSchema()); + assertEquals("For schema 'test': Field 'metadata' of type '" + type + "' cannot be an attribute." + + " Instead specify the struct fields to be searchable as attribute", + exception.getMessage()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java index 188017e0af1..60adf7cbaf0 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java @@ -30,7 +30,7 @@ public class IdTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField uri = new SDField(document, "URI", DataType.URI); - uri.parseIndexingScript("{ summary | index }"); + uri.parseIndexingScript(schema.getName(), "{ summary | index }"); document.addField(uri); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java index 766c19bd01c..596ea99c36d 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java @@ -6,7 +6,6 @@ import com.yahoo.document.Field; import com.yahoo.document.StructDataType; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -194,7 +193,7 @@ public class IndexSchemaTestCase { private static void assertFlat(Field fieldToFlatten, Field... expectedFields) { List<Field> actual = new LinkedList<>(IndexSchema.flattenField(fieldToFlatten)); - List<Field> expected = new LinkedList<>(Arrays.asList(expectedFields)); + List<Field> expected = new LinkedList<>(List.of(expectedFields)); Collections.sort(actual); Collections.sort(expected); for (Field field : actual) { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java index 472fa58230e..b1dc2e66bee 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java @@ -65,7 +65,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { derive("inheritdiamond", builder, builder.getSchema("child")); assertCorrectConfigFiles("inheritdiamond"); } - List<String> files = Arrays.asList("grandparent.sd", "mother.sd", "father.sd", "child.sd"); + List<String> files = List.of("grandparent.sd", "mother.sd", "father.sd", "child.sd"); File outDir = newFolder(tmpDir, "out"); for (int startIdx = 0; startIdx < files.size(); ++startIdx) { var builder = new ApplicationBuilder(new TestProperties()); @@ -159,7 +159,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { Schema parentSchema = new Schema("parent", MockApplicationPackage.createEmpty()); parentSchema.addDocument(parent); SDField prefixed = parent.addField("prefixed", DataType.STRING); - prefixed.parseIndexingScript("{ index }"); + prefixed.parseIndexingScript(parentSchema.getName(), "{ index }"); prefixed.addIndex(new Index("prefixed", true)); SDDocumentType child = new SDDocumentType("child"); @@ -175,7 +175,7 @@ public class InheritanceTestCase extends AbstractExportingTestCase { @Test void testInheritStructDiamondNew() throws IOException, ParseException { String dir = "src/test/derived/declstruct/"; - List<String> files = Arrays.asList("common.sd", "foo.sd", "bar.sd", "foobar.sd"); + List<String> files = List.of("common.sd", "foo.sd", "bar.sd", "foobar.sd"); var builder = new ApplicationBuilder(new TestProperties()); for (String fileName : files) { builder.addSchemaFile(dir + fileName); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java index 97a3f06ac64..505960256b8 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java @@ -15,7 +15,7 @@ import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import java.util.Set; import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; @@ -36,7 +36,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); + field1.parseIndexingScript(schema.getName(), "{ index }"); field1.setLiteralBoost(20); RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); @@ -50,7 +50,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { derived.getAttributeFields(); // TODO: assert content // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", + assertIndexing(List.of("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", "clear_state | guard { input a | tokenize | index a_literal; }"), schema); @@ -69,7 +69,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("literalboost"); schema.addDocument(document); SDField field1 = document.addField("a", DataType.STRING); - field1.parseIndexingScript("{ index }"); + field1.parseIndexingScript(schema.getName(), "{ index }"); RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); @@ -78,7 +78,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { DerivedConfiguration derived = new DerivedConfiguration(schema, rankProfileRegistry); // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", + assertIndexing(List.of("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", "clear_state | guard { input a | tokenize | index a_literal; }"), schema); @@ -95,15 +95,15 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDDocumentType document = new SDDocumentType("msb"); schema.addDocument(document); SDField field1 = document.addField("title", DataType.STRING); - field1.parseIndexingScript("{ summary | index }"); + field1.parseIndexingScript(schema.getName(), "{ summary | index }"); field1.setLiteralBoost(20); SDField field2 = document.addField("body", DataType.STRING); - field2.parseIndexingScript("{ summary | index }"); + field2.parseIndexingScript(schema.getName(), "{ summary | index }"); field2.setLiteralBoost(20); schema = ApplicationBuilder.buildFromRawSchema(schema, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(schema, rankProfileRegistry); - assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", + assertIndexing(List.of("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", "clear_state | guard { input body | tokenize | index body_literal; }"), diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java index 593f26eb074..8ae6ada7a63 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java @@ -12,13 +12,11 @@ import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.TemporarySDField; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -63,7 +61,7 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { private static Schema createSchema(String name, Map<String, Schema> schemas) { Schema schema = new Schema(name, MockApplicationPackage.createEmpty()); SDDocumentType document = new SDDocumentType(name); - document.setDocumentReferences(new DocumentReferences(emptyMap())); + document.setDocumentReferences(new DocumentReferences(Map.of())); schema.addDocument(document); schemas.put(schema.getName(), schema); return schema; @@ -74,9 +72,8 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { } private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) { - inputNames.sort((a, b) -> a.compareTo(b)); Map<String, Schema> schemas = createSchemas(); - List<Schema> inputSchemas = inputNames.stream() + List<Schema> inputSchemas = inputNames.stream().sorted() .map(schemas::get) .map(Objects::requireNonNull) .toList(); @@ -88,7 +85,6 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { assertEquals(expectedSearchOrder, actualSearchOrder); } - @SuppressWarnings("deprecation") private static void createDocumentReference(Schema from, Schema to, String refFieldName) { SDDocumentType fromDocument = from.getDocument(); SDField refField = new TemporarySDField(fromDocument, refFieldName, NewDocumentReferenceDataType.forDocumentName(to.getName())); @@ -102,44 +98,44 @@ public class SchemaOrdererTestCase extends AbstractSchemaTestCase { @Test void testPerfectOrderingIsKept() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); } @Test void testOneLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); } @Test void testMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); } @Test void testAloneIsKeptInPlaceWithMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); } @Test void testPartialMultiLevelReordering() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); } @Test void testMultilevelReorderingAccrossHierarchies() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), - Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), + List.of("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); } @Test void referees_are_ordered_before_referrer() { - assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), - Arrays.asList("accessory-pc", "daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); + assertOrder(List.of("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), + List.of("accessory-pc", "daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index a27bc824b45..63510785ca5 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -172,7 +172,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { schema.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); - field.parseIndexingScript("{ attribute | summary }"); + field.parseIndexingScript(schema.getName(), "{ attribute | summary }"); new Processing().process(schema, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false, Set.of()); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java index d2b3acc9a1e..cdfe376416b 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java @@ -32,7 +32,7 @@ public class TypeConversionTestCase extends AbstractSchemaTestCase { SDDocumentType document = new SDDocumentType("test"); schema.addDocument(document); SDField a = new SDField(document, "a", DataType.STRING); - a.parseIndexingScript("{ index }"); + a.parseIndexingScript(schema.getName(), "{ index }"); document.addField(a); new Processing().process(schema, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), diff --git a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java index 852f567ccfa..a90b4fa8d9f 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -46,7 +46,7 @@ public class VsmFieldsTestCase { void reference_type_field_is_unsearchable() { Schema schema = createSchema(); SDField field = new TemporarySDField(schema.getDocument(), "ref_field", NewDocumentReferenceDataType.forDocumentName("parent_type")); - field.parseIndexingScript("{ summary }"); + field.parseIndexingScript(schema.getName(), "{ summary }"); schema.getDocument().addField(field); VsmfieldsConfig cfg = vsmfieldsConfig(schema); @@ -59,7 +59,7 @@ public class VsmFieldsTestCase { private void testIndexMatching(Matching matching, VsmfieldsConfig.Fieldspec.Normalize.Enum normalize, String arg1) { Schema schema = createSchema(); SDField field = new TemporarySDField(schema.getDocument(), "f", DataType.STRING); - field.parseIndexingScript("{ index }"); + field.parseIndexingScript(schema.getName(), "{ index }"); field.setMatching(matching); schema.getDocument().addField(field); VsmfieldsConfig cfg = vsmfieldsConfig(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java index f09a95b89a0..3da6f250853 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java @@ -19,7 +19,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -63,7 +63,7 @@ public class AddDataTypeAndTransformToSummaryOfImportedFieldsTest { SDField targetField = new SDField(doc, "target_field", DataType.INT); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + return new ImportedFields(Map.of(fieldName, importedField)); } private static DocumentSummary createDocumentSummary(String fieldName, Schema schema) { diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java index aad6df62993..27e9aae04b5 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java @@ -43,9 +43,9 @@ public class AddExtraFieldsToDocumentTest { assertSummary(schema, "foo", "my_a", SummaryTransform.COPY, "a"); assertSummary(schema, "foo", "my_b", SummaryTransform.COPY, "b"); assertSummary(schema, "foo", "my_c", SummaryTransform.ATTRIBUTE, "c"); - // Extra fields should still be created - assertField(schema, "my_a", DataType.STRING); - assertField(schema,"my_b", DataType.INT); + // Extra fields should not be created + assertNull(schema.getDocument().getField("my_a")); + assertNull(schema.getDocument().getField("my_b")); assertNull(schema.getDocument().getField("my_c")); } @@ -78,11 +78,4 @@ public class AddExtraFieldsToDocumentTest { assertEquals(transform, field.getTransform()); assertEquals(source, field.getSingleSource()); } - - private void assertField(Schema schema, String name, DataType type) { - var field = schema.getDocument().getField(name); - assertNotNull(field); - assertEquals(field.getDataType(), type); - } - } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java index ff7e43b2936..b9685d9a4ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java @@ -37,11 +37,11 @@ public class ImplicitSchemaFieldsTestCase extends AbstractSchemaTestCase { SDDocumentType docType = schema.getDocument(); assertNotNull(docType); assertNotNull(docType.getField("foo")); - assertNotNull(docType.getField("bar")); - assertNotNull(docType.getField("cox")); + assertNull(docType.getField("bar")); + assertNull(docType.getField("cox")); assertNotNull(docType.getField("mytags")); - assertNotNull(docType.getField("alltags")); - assertEquals(5, docType.getFieldCount()); + assertNull(docType.getField("alltags")); + assertEquals(2, docType.getFieldCount()); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java index 135a9fa295a..e4d1b5da29e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java @@ -34,9 +34,9 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { assertNotNull(docType); assertField(docType, "doc_str", DataType.STRING); - assertField(docType, "doc_str_sum", DataType.STRING); + assertNoField(docType, "doc_str_sum"); assertField(docType, "doc_uri", DataType.URI); - assertField(docType, "docsum_str", DataType.STRING); + assertNoField(docType, "docsum_str"); } @SuppressWarnings({ "UnusedDeclaration" }) @@ -60,6 +60,11 @@ public class ImplicitStructTypesTestCase extends AbstractSchemaTestCase { assertTrue(field instanceof SDField); } + private static void assertNoField(SDDocumentType docType, String fieldName) { + var field = getSecretField(docType, fieldName); + assertNull(field); + } + private static Field getSecretField(SDDocumentType docType, String fieldName) { for (Field field : docType.fieldSet()) { if (field.getName().equals(fieldName)) { diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index f56d2b21a2d..d5af996bd59 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -78,7 +78,7 @@ public class IndexingOutputsTestCase { """; var builder = ApplicationBuilder.createFromString(sd); var schema = builder.getSchema(); - assertEquals("{ input foo | summary baz | summary bar; }", + assertEquals("{ input foo | summary bar; }", schema.getConcreteField("bar").getIndexingScript().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index c2cc28ea6b3..de99d46b9ca 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; @@ -64,7 +64,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field.addSummaryField(createStaticSummaryField(field, "test")); field.addSummaryField(createStaticSummaryField(field, "other")); field.addSummaryField(createDynamicSummaryField(field, "dyn2")); - assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary other | " + + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | " + "summary test | index test; }", field); } @@ -105,7 +105,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { @Test void testDerivingFromSimple() throws Exception { - assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", + assertIndexing(List.of("clear_state | guard { input access | attribute access; }", "clear_state | guard { input category | split \";\" | attribute category_arr; }", "clear_state | guard { input category | tokenize | index category; }", "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", @@ -113,7 +113,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | index description; }", "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", - "clear_state | guard { input longdesc | summary longdesc | summary longstat; }", + "clear_state | guard { input longdesc | summary longdesc; }", "clear_state | guard { input measurement | attribute measurement | summary measurement; }", "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", "clear_state | guard { input popularity | attribute popularity; }", @@ -127,8 +127,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { @Test void testIndexRewrite() throws Exception { assertIndexing( - Arrays.asList("clear_state | guard { input title_src | lowercase | normalize | " + - " tokenize | index title; }", + List.of("clear_state | guard { input title_src | lowercase | normalize | tokenize | index title; }", "clear_state | guard { input title_src | summary title_s; }"), ApplicationBuilder.buildFromFile("src/test/examples/indexrewrite.sd")); } @@ -151,7 +150,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { void requireThatMaxTermOccurrencesIsPropagated() { var field = new SDField("test", DataType.STRING); field.getMatching().maxTermOccurrences(10); - field.parseIndexingScript("{ summary | index }"); + field.parseIndexingScript("test", "{ summary | index }"); assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-occurrences:10 | summary test | index test; }", field); } @@ -173,14 +172,14 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { private static SDField createField(String name, DataType type, String script) { SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); + field.parseIndexingScript("test", script); return field; } private static SDField createPredicateField( String name, DataType type, String script, int arity, OptionalLong lower_bound, OptionalLong upper_bound) { SDField field = new SDField(null, name, type); - field.parseIndexingScript(script); + field.parseIndexingScript("test", script); Index index = new Index("foo"); index.setBooleanIndexDefiniton(new BooleanIndexDefinition( OptionalInt.of(arity), lower_bound, upper_bound, OptionalDouble.empty())); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index 71c91533f54..4053834784f 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Arrays; +import java.util.List; import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -154,7 +154,7 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { @Test void testExtraField() throws IOException, ParseException { assertIndexing( - Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", + List.of("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), ApplicationBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java index af275feffed..129d65584ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java @@ -36,8 +36,11 @@ public class ParentChildSearchModel { } protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) { + return createField(repo, repo.getName(), name, dataType, indexingScript); + } + protected static TemporarySDField createField(SDDocumentType repo, String schemaName, String name, DataType dataType, String indexingScript) { TemporarySDField result = new TemporarySDField(repo, name, dataType); - result.parseIndexingScript(indexingScript); + result.parseIndexingScript(schemaName, indexingScript); return result; } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java index 007006bf6d3..87a66046fe5 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java @@ -13,8 +13,8 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -27,7 +27,7 @@ public class PositionTestCase { @Test void inherited_position_zcurve_field_is_not_added_to_document_fieldset() throws Exception { - ApplicationBuilder sb = ApplicationBuilder.createFromFiles(Arrays.asList( + ApplicationBuilder sb = ApplicationBuilder.createFromFiles(List.of( "src/test/examples/position_base.sd", "src/test/examples/position_inherited.sd")); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java index 9213f97fd9f..6cbd8faf3b0 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -19,7 +19,7 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.Tensor; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -94,16 +94,17 @@ public class RankingExpressionWithTransformerTokensTestCase { MockApplicationPackage application = (MockApplicationPackage) MockApplicationPackage.createEmpty(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfileRegistry queryProfileRegistry = application.getQueryProfiles(); - String sdContent = "search test {\n" + - " document test {}\n" + - " rank-profile my_profile inherits default {}\n" + - "}"; + String sdContent = """ + search test { + document test {} + rank-profile my_profile inherits default {} + }"""; ApplicationBuilder schemaBuilder = new ApplicationBuilder(application, new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfileRegistry); schemaBuilder.addSchema(sdContent); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); RankProfile rp = rankProfileRegistry.get(schema, "my_profile"); - return new RankProfileTransformContext(rp, queryProfileRegistry, Collections.emptyMap(), null, Collections.emptyMap(), Collections.emptyMap()); + return new RankProfileTransformContext(rp, queryProfileRegistry, Map.of(), null, Map.of(), Map.of()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java index 0f16330ce11..a602503a71c 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java @@ -21,7 +21,6 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -102,12 +101,12 @@ public class RankingExpressionsTestCase extends AbstractSchemaTestCase { { AttributeFields attributes = new AttributeFields(schema); - verifyProfile(rankProfileRegistry.get(schema, "base"), Arrays.asList("large_f", "large_m"), - Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), + verifyProfile(rankProfileRegistry.get(schema, "base"), List.of("large_f", "large_m"), + List.of(new Pair<>("rankingExpression(large_f).expressionName", "base.large_f"), new Pair<>("rankingExpression(large_m).expressionName", "base.large_m")), largeExpressions, queryProfiles, models, attributes, properties); - for (String child : Arrays.asList("child_a", "child_b")) { - verifyProfile(rankProfileRegistry.get(schema, child), Arrays.asList("large_f", "large_m", "large_local_f", "large_local_m"), - Arrays.asList(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), + for (String child : List.of("child_a", "child_b")) { + verifyProfile(rankProfileRegistry.get(schema, child), List.of("large_f", "large_m", "large_local_f", "large_local_m"), + List.of(new Pair<>("rankingExpression(large_f).expressionName", child + ".large_f"), new Pair<>("rankingExpression(large_m).expressionName", child + ".large_m"), new Pair<>("rankingExpression(large_local_f).expressionName", child + ".large_local_f"), new Pair<>("rankingExpression(large_local_m).expressionName", child + ".large_local_m"), new Pair<>("vespa.rank.firstphase", "rankingExpression(firstphase)"), new Pair<>("rankingExpression(firstphase).expressionName", child + ".firstphase")), largeExpressions, queryProfiles, models, attributes, properties); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java index be72d2b12fa..8585a206d77 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -65,7 +65,7 @@ public class ValidateFieldTypesTest { SDField targetField = new SDField(targetSchema.getDocument(), "target_field", dataType); DocumentReference documentReference = new DocumentReference(new Field("reference_field"), targetSchema); ImportedField importedField = new ImportedSimpleField(fieldName, documentReference, targetField); - return new ImportedFields(Collections.singletonMap(fieldName, importedField)); + return new ImportedFields(Map.of(fieldName, importedField)); } private static DocumentSummary createDocumentSummary(String fieldName, DataType dataType, Schema schema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 326fb633877..8265dac9751 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -212,6 +212,27 @@ public class DedicatedAdminV4Test { METRICS_PROXY_CONTAINER.serviceName, LOGSERVER_CONTAINER.serviceName); } + @Test + void testOtelServiceWhenFeatureFlagEnabled() throws Exception { + String services = "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " </admin>" + + "</services>"; + + VespaModel model = createModel(hosts, services, new DeployState.Builder() + .zone(new Zone(SystemName.Public, Environment.dev, RegionName.defaultName())) + .properties(new TestProperties() + .setLogserverOtelCol(true) + .setHostedVespa(true))); + assertEquals(1, model.getHosts().size()); + // Should create a logserver container on the same node as logserver + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", "opentelemetrycollector", + METRICS_PROXY_CONTAINER.serviceName, LOGSERVER_CONTAINER.serviceName); + } + private Set<String> serviceNames(VespaModel model, String hostname) { SentinelConfig config = model.getConfig(SentinelConfig.class, hostname); return config.service().stream().map(SentinelConfig.Service::name).collect(Collectors.toSet()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 88e1ba7a1a6..94dd0f367a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -8,6 +8,8 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import org.junit.jupiter.api.Test; +import java.util.List; + import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; @@ -21,7 +23,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.c import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; -import static java.util.Collections.singleton; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -72,7 +73,7 @@ public class MetricsConsumersTest { void vespa_consumer_can_be_amended_via_admin_object() { VespaModel model = getModel(servicesWithAdminOnly(), hosted); var additionalMetric = new Metric("additional-metric"); - model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); + model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", List.of(additionalMetric))); ConsumersConfig config = consumersConfigFromModel(model); assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java new file mode 100644 index 00000000000..7c4968aac84 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -0,0 +1,22 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.otel; + +import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import org.junit.jupiter.api.Test; +import java.util.List; +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author arnej + */ +public class OpenTelemetryConfigGeneratorTest { + + @Test + void testBuildsYaml() { + var generator = new OpenTelemetryConfigGenerator(null); + generator.addStatePorts(List.of(new StatePortInfo("localhost", 19098, "config-sentinel", "sentinel"))); + String yaml = generator.generate(); + assertTrue(yaml.contains("sentinel")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java index 6fbca76ccbc..72230a580d7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudClientsValidatorTest.java @@ -18,8 +18,6 @@ class CloudClientsValidatorTest { @Test void logs_deployment_warning_on_certificate_with_empty_sequence_of_extensions() { - // Test should fail on BouncyCastle 1.77 or later - var logger = new DeployLoggerStub(); var state = new DeployState.Builder().deployLogger(logger).build(); var cert = readTestCertificate("cert-with-empty-sequence-of-extensions.pem"); @@ -30,8 +28,7 @@ class CloudClientsValidatorTest { "The certificate's ASN.1 structure contains an empty sequence of extensions, " + "which is a violation of the ASN.1 specification. " + "Please update the application package with a new certificate, " + - "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. " + - "Such certificate will no longer be accepted in near future."; + "e.g by generating a new one using the Vespa CLI `$ vespa auth cert`. "; assertEquals(expected, logger.getLast().message); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index b2291099b44..ae1db366c9f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.logging.Level; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -67,7 +68,9 @@ public class ComplexFieldsValidatorTestCase { "}", "}")); }); - assertTrue(exception.getMessage().contains(getExpectedMessage("docTopics (docTopics.topics, docTopics.topics.id, docTopics.topics.label)"))); + assertEquals("For schema 'test': Field 'docTopics.topics' of type 'array<topic>' cannot be an attribute." + + " Instead specify the struct fields to be searchable as attribute", + exception.getMessage()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java index 19be886d3e5..c5ef2238f9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/PublicApiBundleValidatorTest.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.io.TempDir; import java.io.File; import java.io.IOException; -import java.util.Arrays; import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; @@ -42,7 +41,7 @@ public class PublicApiBundleValidatorTest { assertThat(output, containsString("uses non-public Vespa APIs: [")); // List of packages should be sorted - List<String> packages = Arrays.asList(output.substring(output.indexOf("[") + 1, output.indexOf("]")).split(", ")); + List<String> packages = List.of(output.substring(output.indexOf("[") + 1, output.indexOf("]")).split(", ")); assertThat(packages, hasSize(2)); assertThat(packages, contains("ai.vespa.lib.non_public", "com.yahoo.lib.non_public")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java index e4b9b45489d..829bd6148a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java @@ -54,7 +54,7 @@ public class StartupCommandChangeValidatorTest { private static MockRoot createRootWithChildren(TreeConfigProducer<?>... children) { MockRoot root = new MockRoot(); - Arrays.asList(children).forEach(root::addChild); + List.of(children).forEach(root::addChild); root.freezeModelTopology(); return root; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 91ba97817d6..8778f0c26c0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.content.utils.SchemaBuilder; import com.yahoo.vespa.model.search.DocumentDatabase; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -32,15 +31,15 @@ public abstract class ContentClusterFixture { public ContentClusterFixture(String entireSd) throws Exception { currentCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(Arrays.asList(entireSd))); + ContentClusterUtils.createMockRoot(List.of(entireSd))); nextCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(Arrays.asList(entireSd))); + ContentClusterUtils.createMockRoot(List.of(entireSd))); } private static ContentCluster createCluster(String sdContent) throws Exception { return new ContentClusterBuilder().build( ContentClusterUtils.createMockRoot( - Arrays.asList(new SchemaBuilder().content(sdContent).build()))); + List.of(new SchemaBuilder().content(sdContent).build()))); } protected DocumentDatabase currentDb() { @@ -65,7 +64,7 @@ public abstract class ContentClusterFixture { } public void assertValidation(VespaConfigChangeAction exp) { - assertValidation(Arrays.asList(exp)); + assertValidation(List.of(exp)); } public void assertValidation(List<VespaConfigChangeAction> exp) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index 684bd619ba1..129649ae1fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeActi import org.junit.jupiter.api.Test; import java.time.Instant; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; @@ -50,7 +49,7 @@ public class DocumentDatabaseChangeValidatorTest { "field f3 type string { indexing: summary } " + "field f4 type array<s> { struct-field s1 { indexing: attribute } }"); Instant.now(); - f.assertValidation(Arrays.asList( + f.assertValidation(List.of( newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1' changed: add attribute aspect"), newRestartAction(ClusterSpec.Id.from("test"), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 44743c4fa3e..26a4f532362 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -14,10 +14,9 @@ import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import org.junit.jupiter.api.Test; import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -108,7 +107,7 @@ public class DocumentTypeChangeValidatorTest { "field f2 type string { indexing: summary } field f1 type int { indexing: summary }"); Instant.now(); Instant.now(); - f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'"), + f.assertValidation(List.of(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'"), newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 'int' -> 'string'"))); } @@ -210,8 +209,8 @@ public class DocumentTypeChangeValidatorTest { new NewDocumentType.Name("mydoc"), headerfields, new FieldSets(Optional.empty()), - Collections.emptySet(), - Collections.emptySet()); + Set.of(), + Set.of()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 6b58cac3f6c..cd54a20523f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -12,7 +12,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; public class IndexingScriptChangeValidatorTest { @@ -126,7 +125,7 @@ public class IndexingScriptChangeValidatorTest { void requireThatMultipleChangesRequireReindexing() throws Exception { new Fixture(FIELD + " { indexing: index } " + FIELD_F2 + " { indexing: index }", FIELD + " { indexing: index \n stemming: none } " + FIELD_F2 + " { indexing: index \n normalizing: none }"). - assertValidation(Arrays.asList(expectedReindexingAction("f1", "stemming: 'best' -> 'none'", + assertValidation(List.of(expectedReindexingAction("f1", "stemming: 'best' -> 'none'", "{ input f1 | tokenize normalize stem:\"BEST\" | index f1; }", "{ input f1 | tokenize normalize | index f1; }"), expectedReindexingAction("f2", "normalizing: 'ACCENT' -> 'NONE'", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index fac82f3e675..924419daeae 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import java.util.Set; @@ -876,8 +875,8 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SCHEMA, - MockApplicationPackage.BOOK_SCHEMA)) + .withSchemas(List.of(MockApplicationPackage.MUSIC_SCHEMA, + MockApplicationPackage.BOOK_SCHEMA)) .build()) .create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 853cfdd9429..0e616661191 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -112,7 +111,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { @Test void multitenant() { - List<ConfigServerSpec> configServerSpecs = Arrays.asList( + List<ConfigServerSpec> configServerSpecs = List.of( new TestProperties.Spec("test1", 19070, 2181), new TestProperties.Spec("test2", 19070, 2181), new TestProperties.Spec("test3", 19070, 2181)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java index 764e31fe13a..c7525dec6a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.model.search.Tuning; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -22,7 +22,7 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { private static Element parseXml(String... xmlLines) { return parse("<tuning>", "<searchnode>", - CollectionUtil.mkString(Arrays.asList(xmlLines), "\n"), + CollectionUtil.mkString(List.of(xmlLines), "\n"), "</searchnode>", "</tuning>"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java index 6b384269910..56f492f3fca 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -98,7 +97,7 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { @Test void ensureSearchChainsExists() { - for (String id : Arrays.asList("provider:1", "source:1@provider:1", "default")) { + for (String id : List.of("provider:1", "source:1@provider:1", "default")) { assertNotNull(getSearchChain(id), "Missing search chain " + id); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/component/ModelTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/component/ModelTest.java new file mode 100644 index 00000000000..7660ae45c3a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/component/ModelTest.java @@ -0,0 +1,33 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.text.XML; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author hmusum + */ +public class ModelTest { + + @Test + void invalid_url(){ + var xml = """ + <component id="bert-embedder" type="bert-embedder"> + <transformer-model url="models/e5-base-v2.onnx" /> + <tokenizer-vocab path="models/vocab.txt"/> + </component> + """; + + var state = new DeployState.Builder().build(); + var element = XML.getDocument(xml).getDocumentElement(); +var exception = assertThrows(IllegalArgumentException.class, + () -> Model.fromXml(state, element, "transformer-model", Set.of())); +org.junit.jupiter.api.Assertions.assertEquals("Invalid url 'models/e5-base-v2.onnx': url has no 'scheme' component", exception.getMessage()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index b933d86655e..4e14fbcd671 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -21,12 +21,9 @@ import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Function; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -46,7 +43,7 @@ public class ConfigserverClusterTest { @Test void zookeeperConfig_only_config_servers_set_hosted() { - TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.emptyList()); + TestOptions testOptions = createTestOptions(List.of("cfg1", "localhost", "cfg3"), List.of()); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2); @@ -57,7 +54,7 @@ public class ConfigserverClusterTest { @Test void zookeeperConfig_with_config_servers_and_zk_ids_hosted() { - TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3)); + TestOptions testOptions = createTestOptions(List.of("cfg1", "localhost", "cfg3"), List.of(4, 2, 3)); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); @@ -68,7 +65,7 @@ public class ConfigserverClusterTest { @Test void zookeeperConfig_self_hosted() { final boolean hostedVespa = false; - TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3), hostedVespa); + TestOptions testOptions = createTestOptions(List.of("cfg1", "localhost", "cfg3"), List.of(4, 2, 3), hostedVespa); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); @@ -80,7 +77,7 @@ public class ConfigserverClusterTest { @Test void zookeeperConfig_uneven_number_of_config_servers_and_zk_ids() { assertThrows(IllegalArgumentException.class, () -> { - TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.singletonList(1)); + TestOptions testOptions = createTestOptions(List.of("cfg1", "localhost", "cfg3"), List.of(1)); getConfig(ZookeeperServerConfig.class, testOptions); }); } @@ -88,7 +85,7 @@ public class ConfigserverClusterTest { @Test void zookeeperConfig_negative_zk_id() { assertThrows(IllegalArgumentException.class, () -> { - TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(1, 2, -1)); + TestOptions testOptions = createTestOptions(List.of("cfg1", "localhost", "cfg3"), List.of(1, 2, -1)); getConfig(ZookeeperServerConfig.class, testOptions); }); } @@ -137,7 +134,7 @@ public class ConfigserverClusterTest { private static <T> void assertZookeeperServerProperty( List<ZookeeperServerConfig.Server> zkServers, Function<ZookeeperServerConfig.Server, T> propertyMapper, T... expectedProperties) { List<T> actualPropertyValues = zkServers.stream().map(propertyMapper).toList(); - List<T> expectedPropertyValues = Arrays.asList(expectedProperties); + List<T> expectedPropertyValues = List.of(expectedProperties); assertEquals(expectedPropertyValues, actualPropertyValues); } @@ -169,7 +166,7 @@ public class ConfigserverClusterTest { } private static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz) { - return getConfig(clazz, createTestOptions(Collections.emptyList(), Collections.emptyList())); + return getConfig(clazz, createTestOptions(List.of(), List.of())); } private static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, TestOptions testOptions) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java index 73c4075a182..3345f935e1d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -29,7 +28,7 @@ public class BlockFeedGlobalEndpointsFilterTest { @Test void does_not_setup_blocking_rule_when_endpoints_empty() { - var filter = new BlockFeedGlobalEndpointsFilter(Collections.emptySet(), true); + var filter = new BlockFeedGlobalEndpointsFilter(Set.of(), true); var config = getConfig(filter); assertEquals(0, config.rule().size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java index 1c5eb16be80..fa09d3c1890 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java @@ -162,6 +162,36 @@ public class CloudTokenDataPlaneFilterTest extends ContainerModelBuilderTestBase assertEquals("Invalid permission 'unknown-permission'. Valid values are 'read' and 'write'.", exception.getMessage()); } + @Test + void fails_on_duplicate_clients() throws IOException { + var certFile = securityFolder.resolve("foo.pem"); + var servicesXml = """ + <container version="1.0"> + <clients> + <client id="mtls" permissions="read,write"> + <certificate file="%1$s"/> + </client> + <client id="mtls" permissions="read,write"> + <certificate file="%1$s"/> + </client> + <client id="token1" permissions="read"> + <token id="my-token"/> + </client> + <client id="token2" permissions="read"> + <token id="my-token"/> + </client> + <client id="token1" permissions="read"> + <token id="my-token"/> + </client> + </clients> + </container> + """.formatted(applicationFolder.toPath().relativize(certFile).toString()); + var clusterElem = DomBuilderTest.parse(servicesXml); + createCertificate(certFile); + var exception = assertThrows(IllegalArgumentException.class, () -> buildModel(Set.of(mtlsEndpoint), defaultTokens, clusterElem)); + assertEquals("Duplicate client ids: [mtls, token1]", exception.getMessage()); + } + private static CloudTokenDataPlaneFilterConfig.Clients.Tokens tokenConfig( String id, Collection<String> fingerprints, Collection<String> accessCheckHashes, Collection<String> expirations) { return new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index a066308b426..a832c54021f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -19,8 +19,8 @@ import org.junit.jupiter.api.BeforeEach; import org.w3c.dom.Element; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -88,7 +88,7 @@ public abstract class ContainerModelBuilderTestBase { private static void generateDefaultSearchChains(ContainerCluster<?> cluster) { ContainerSearch search = cluster.getSearch(); if (search != null) - search.initializeSearchChains(Collections.emptyMap()); + search.initializeSearchChains(Map.of()); } protected ComponentsConfig componentsConfig() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java index 4efffc8310a..fb1e176f707 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java @@ -158,7 +158,7 @@ public class EmbedderTestCase { var model = loadModel(Path.fromString("src/test/cfg/application/embed/"), false); var cluster = model.getContainerClusters().get("container"); var embedderCfg = assertBertEmbedderComponentPresent(cluster); - assertEquals("application-url", modelReference(embedderCfg, "transformerModel").url().orElseThrow().value()); + assertEquals("https://my/url/model.onnx", modelReference(embedderCfg, "transformerModel").url().orElseThrow().value()); assertEquals("files/vocab.txt", modelReference(embedderCfg, "tokenizerVocab").path().orElseThrow().value()); assertEquals("", embedderCfg.transformerTokenTypeIds()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 10adca12f91..46097da434e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -19,13 +19,12 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -218,18 +217,17 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } private String verifyLogMessage(TestLogger logger, String... invalidOptions) { - List<String> strings = Arrays.asList(invalidOptions.clone()); + List<String> strings = List.of(invalidOptions); // Verify that nothing is logged if there are no invalid options if (strings.isEmpty()) { - assertEquals(0, logger.msgs.size(), logger.msgs.size() > 0 ? logger.msgs.get(0).getSecond() : ""); + assertEquals(0, logger.msgs.size(), !logger.msgs.isEmpty() ? logger.msgs.get(0).getSecond() : ""); return null; } - assertTrue(logger.msgs.size() > 0, "Expected 1 or more log messages for invalid JM options, got none"); + assertFalse(logger.msgs.isEmpty(), "Expected 1 or more log messages for invalid JM options, got none"); Pair<Level, String> firstOption = logger.msgs.get(0); assertEquals(Level.WARNING, firstOption.getFirst()); - Collections.sort(strings); return firstOption.getSecond(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 1ba5bebea7e..786caa4b317 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; -import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.search.DispatchConfig; @@ -43,8 +42,6 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -57,6 +54,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -72,7 +70,7 @@ public class ContentClusterTest extends ContentBaseTest { @Test void testHierarchicRedundancy() { - ContentCluster cc = parse("" + + ContentCluster cc = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + " <engine>" + @@ -947,8 +945,8 @@ public class ContentClusterTest extends ContentBaseTest { .properties(props); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), - Collections.emptyList(), deployStateBuilder) : - ContentClusterUtils.createMockRoot(Collections.emptyList(), deployStateBuilder); + List.of(), deployStateBuilder) : + ContentClusterUtils.createMockRoot(List.of(), deployStateBuilder); ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); @@ -1104,8 +1102,8 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, config.cluster().size()); - assertClusterHasBucketSpaceMappings(config, "foo_c", Arrays.asList("bunnies", "hares"), Collections.emptyList()); - assertClusterHasBucketSpaceMappings(config, "bar_c", Collections.emptyList(), Collections.singletonList("rabbits")); + assertClusterHasBucketSpaceMappings(config, "foo_c", List.of("bunnies", "hares"), List.of()); + assertClusterHasBucketSpaceMappings(config, "bar_c", List.of(), List.of("rabbits")); } @Test @@ -1263,7 +1261,7 @@ public class ContentClusterTest extends ContentBaseTest { } @Test - void verifyt_max_tls_size() throws Exception { + void verify_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); @@ -1490,6 +1488,24 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + @Test + void node_distribution_key_outside_legal_range_is_disallowed() { + // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal + // sentinel value that must never be used by actual nodes. + for (int distKey : List.of(-1, 65535, 65536, 100000)) { + assertThrows(IllegalArgumentException.class, () -> + parse(""" + <content version="1.0" id="storage"> + <documents/> + <redundancy>1</redundancy> + <group> + <node hostalias='mockhost' distribution-key='%d' /> + </group> + </content>""".formatted(distKey) + )); + } + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 55ec9f4efe2..8c08b9054a4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -60,12 +59,12 @@ public class ContentSchemaClusterTest { private static ContentClusterBuilder createClusterBuilderWithGlobalType() { return new ContentClusterBuilder() - .docTypes(Arrays.asList(DocType.indexGlobal("global"), DocType.index("regular"))); + .docTypes(List.of(DocType.indexGlobal("global"), DocType.index("regular"))); } private static ContentClusterBuilder createClusterBuilderWithOnlyDefaultTypes() { return new ContentClusterBuilder() - .docTypes(Arrays.asList(DocType.index("marve"), DocType.index("fleksnes"))); + .docTypes(List.of(DocType.index("marve"), DocType.index("fleksnes"))); } private static ProtonConfig getProtonConfig(ContentCluster cluster) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java index 6066a27cff5..e671a47b36c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GlobalDistributionValidatorTest.java @@ -11,8 +11,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Stream; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; import static java.util.stream.Collectors.toSet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -25,7 +23,7 @@ public class GlobalDistributionValidatorTest { @Test void validation_succeeds_on_no_documents() { new GlobalDistributionValidator() - .validate(emptyMap(), emptySet()); + .validate(Map.of(), Set.of()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index b41d92bd63f..901c307723e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -489,7 +488,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { private DocprocClusterSpec(String name, DocprocChainSpec ... chains) { this.name = name; - this.chains.addAll(Arrays.asList(chains)); + this.chains.addAll(List.of(chains)); } } @@ -500,7 +499,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { private DocprocChainSpec(String name, String ... inherits) { this.name = name; - this.inherits.addAll(Arrays.asList(inherits)); + this.inherits.addAll(List.of(inherits)); } } @@ -542,7 +541,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } public static List<String> generateSchemas(String ... sdNames) { - return generateSchemas(Arrays.asList(sdNames)); + return generateSchemas(List.of(sdNames)); } public static List<String> generateSchemas(List<String> sdNames) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java index a6fa0334668..2fb9a67a9ae 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.documentmodel.NewDocumentType; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -40,13 +38,13 @@ public class ReservedDocumentTypeNameValidatorTest { @Test void exception_is_not_thrown_on_unreserved_name() { ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); - validator.validate(asDocTypeMapping(Collections.singletonList("foo"))); + validator.validate(asDocTypeMapping(List.of("foo"))); } @Test void validation_is_case_insensitive() { ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); - Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(Arrays.asList("NULL", "True", "anD"))); + Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(List.of("NULL", "True", "anD"))); try { validator.validate(orderedDocTypes); fail(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 0dd03d258b9..f56e642edd7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -336,6 +336,23 @@ public class StorageClusterTest { assertEquals(3.0, config.async_operation_throttler().resize_rate(), 0.0001); } + private void verifyMaxFeedOpBatchSize(int expected, Integer flagValue) { + var props = new TestProperties(); + if (flagValue != null) { + props.setPersistenceThreadMaxFeedOpBatchSize(flagValue); + } + var config = filestorConfigFromProducer(simpleCluster(props)); + assertEquals(expected, config.max_feed_op_batch_size()); + } + + @Test + void persistence_max_feed_op_batch_size_is_controlled_by_feature_flag() { + // TODO update default once rolled out and tested + verifyMaxFeedOpBatchSize(1, null); + verifyMaxFeedOpBatchSize(1, 1); + verifyMaxFeedOpBatchSize(1234, 1234); + } + @Test void testCapacity() { String xml = joinLines( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java index c5a645bd863..06794a3c1f3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java @@ -5,8 +5,6 @@ import com.yahoo.documentmodel.NewDocumentType; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -20,24 +18,24 @@ public class TopologicalDocumentTypeSorterTest { @Test void require_that_types_without_references_are_returned_in_input_order() { - assertOrder(Arrays.asList("a"), new DocumentTypesBuilder().add("a")); - assertOrder(Arrays.asList("a", "c", "b"), + assertOrder(List.of("a"), new DocumentTypesBuilder().add("a")); + assertOrder(List.of("a", "c", "b"), new DocumentTypesBuilder().add("a").add("c").add("b")); } @Test void require_that_types_with_references_are_sorted_in_topological_order() { - assertOrder(Arrays.asList("b", "a"), new DocumentTypesBuilder() - .add("a", Arrays.asList("b")) + assertOrder(List.of("b", "a"), new DocumentTypesBuilder() + .add("a", List.of("b")) .add("b")); - assertOrder(Arrays.asList("c", "b", "a"), new DocumentTypesBuilder() - .add("a", Arrays.asList("b", "c")) - .add("b", Arrays.asList("c")) + assertOrder(List.of("c", "b", "a"), new DocumentTypesBuilder() + .add("a", List.of("b", "c")) + .add("b", List.of("c")) .add("c")); - assertOrder(Arrays.asList("b", "a", "d", "c"), new DocumentTypesBuilder() - .add("a", Arrays.asList("b")) + assertOrder(List.of("b", "a", "d", "c"), new DocumentTypesBuilder() + .add("a", List.of("b")) .add("b") - .add("c", Arrays.asList("d")) + .add("c", List.of("d")) .add("d")); } @@ -52,7 +50,7 @@ public class TopologicalDocumentTypeSorterTest { private final List<NewDocumentType> result = new ArrayList<>(); public DocumentTypesBuilder add(String docTypeName) { - return add(docTypeName, Collections.emptyList()); + return add(docTypeName, List.of()); } public DocumentTypesBuilder add(String docTypeName, List<String> docTypeNameReferences) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilderTest.java index 3bd7b7a4c1a..1043ddd2c0b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/GlobalDistributionBuilderTest.java @@ -6,9 +6,7 @@ import com.yahoo.text.XML; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -35,7 +33,7 @@ public class GlobalDistributionBuilderTest { " <document type=\"" + GLOBAL_2.getName() + "\" global=\"true\"/>" + "</documents>"; - Set<NewDocumentType> expectedResult = new HashSet<>(Arrays.asList(GLOBAL_1, GLOBAL_2)); + Set<NewDocumentType> expectedResult = Set.of(GLOBAL_1, GLOBAL_2); Set<NewDocumentType> actualResult = builder.build(new ModelElement(XML.getDocument(documentsElement).getDocumentElement())); assertEquals(expectedResult, actualResult); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java index 61ce5172ceb..277b07023e6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -21,7 +20,7 @@ public class ContentClusterBuilder { private String name = "mycluster"; private int redundancy = 1; private int searchableCopies = 1; - private List<DocType> docTypes = Arrays.asList(DocType.index("test")); + private List<DocType> docTypes = List.of(DocType.index("test")); private String groupXml = getSimpleGroupXml(); private Optional<String> dispatchXml = Optional.empty(); private Optional<Double> protonDiskLimit = Optional.empty(); @@ -54,8 +53,8 @@ public class ContentClusterBuilder { } public ContentClusterBuilder docTypes(String ... docTypes) { - this.docTypes = Arrays.asList(docTypes).stream(). - map(type -> DocType.index(type)). + this.docTypes = Arrays.stream(docTypes). + map(DocType::index). toList(); return this; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index a8c67ebf0a3..360ad19b710 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.content.cluster.ContentCluster; import org.w3c.dom.Document; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -67,7 +66,7 @@ public class ContentClusterUtils { ConfigModelContext context = ConfigModelContext.create(applicationType, root.getDeployState(), null,null, root, null); - return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); + return new ContentCluster.Builder(admin).build(List.of(), context, doc.getDocumentElement()); } public static ContentCluster createCluster(String clusterXml, List<String> schemas, DeployState.Builder deployStateBuilder) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java index 3ffb33a47c8..ce05fa27e2e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.utils; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -46,7 +45,7 @@ public class DocType { } public static String listToXml(DocType... docTypes) { - return listToXml(Arrays.asList(docTypes)); + return listToXml(List.of(docTypes)); } public static String listToXml(List<DocType> docTypes) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java index 304f3dc426f..55ad9149a17 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java @@ -38,8 +38,7 @@ public class SchemaBuilder { } public static List<String> createSchemas(String ... docTypes) { - return Arrays.asList(docTypes) - .stream() + return Arrays.stream(docTypes) .map(type -> new SchemaBuilder().name(type).build()) .toList(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index f9999a30869..7809a97f85c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -41,8 +41,8 @@ public class SchemaClusterTest { SDDocumentType sdt1 = new SDDocumentType("s1"); Schema schema1 = new Schema("s1", MockApplicationPackage.createEmpty()); SDField f1 = new SDField(sdt1, "f1", DataType.STRING); - f1.addAttribute(new Attribute("f1", DataType.STRING)); - f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); + f1.addAttribute(new Attribute(schema1.getName(), f1.getName(), "f1", DataType.STRING)); + f1.setIndexingScript("s1", new ScriptExpression(new StatementExpression(new AttributeExpression("f1")))); sdt1.addField(f1); schema1.addDocument(sdt1); @@ -50,8 +50,8 @@ public class SchemaClusterTest { SDDocumentType sdt2 = new SDDocumentType("s2"); Schema schema2 = new Schema("s2", MockApplicationPackage.createEmpty()); SDField f2 = new SDField(sdt2, "f2", DataType.STRING); - f2.addAttribute(new Attribute("f2", DataType.STRING)); - f2.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); + f2.addAttribute(new Attribute(schema2.getName(), f2.getName(), "f2", DataType.STRING)); + f2.setIndexingScript("s2", new ScriptExpression(new StatementExpression(new AttributeExpression("f2")))); sdt2.addField(f2); schema2.addDocument(sdt2); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 8502bfa92f4..672bba83e87 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -102,7 +102,7 @@ public class SchemaInfoTestCase { var schemaInfoTester = new SchemaInfoTester(); var schema = schemaInfoTester.createSchema("test"); var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); - var attribute = field.addAttribute(new Attribute("f1Attribute", field.getDataType())); + var attribute = field.addAttribute(new Attribute(schema.getName(), field.getName(), "f1Attribute", field.getDataType())); attribute.getAliases().add("a1"); attribute.getAliases().add("a2"); assertEquals(""" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java index 7efacd6b5ad..e17db551798 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -181,7 +180,7 @@ public class SchemaTester { } public static List<String> generateSchemas(String schemaContent, String rankProfile, String ... schemaNames) { - return generateSchemas(schemaContent, rankProfile, Arrays.asList(schemaNames)); + return generateSchemas(schemaContent, rankProfile, List.of(schemaNames)); } public static List<String> generateSchemas(String schemaContent, String rankProfile, List<String> schemaNames) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java index 41003f03c0d..b9be56d8d69 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ApiConfigModel.java @@ -10,7 +10,6 @@ import org.w3c.dom.Element; import org.w3c.dom.NodeList; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -50,7 +49,7 @@ public class ApiConfigModel extends ConfigModel { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("api")); + return List.of(ConfigModelId.fromName("api")); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java index 61bd3d885f5..d9be865bc7d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/SimpleConfigModel.java @@ -10,7 +10,6 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -20,8 +19,8 @@ import java.util.List; */ public class SimpleConfigModel extends ConfigModel implements TestApi { - private List<SimpleService> simpleServices = new ArrayList<>(); - private List<ParentService> parentServices = new ArrayList<>(); + private final List<SimpleService> simpleServices = new ArrayList<>(); + private final List<ParentService> parentServices = new ArrayList<>(); public SimpleConfigModel(ConfigModelContext modelContext) { super(modelContext); @@ -43,7 +42,7 @@ public class SimpleConfigModel extends ConfigModel implements TestApi { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("simple")); + return List.of(ConfigModelId.fromName("simple")); } @Override @@ -54,11 +53,10 @@ public class SimpleConfigModel extends ConfigModel implements TestApi { NodeList childNodes = spec.getChildNodes(); for (int i=0; i < childNodes.getLength(); i++) { Node child = childNodes.item(i); - if (! (child instanceof Element)) { + if (! (child instanceof Element e)) { // skip #text and #comment nodes continue; } - Element e = (Element)child; String service = e.getTagName(); if (service.equals("simpleservice")) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index b9bc34688b7..ce373e558ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.test.utils; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -50,7 +49,7 @@ public class ApplicationPackageUtils { } public static List<String> generateSchemas(String ... sdNames) { - return generateSchemas(Arrays.asList(sdNames)); + return generateSchemas(List.of(sdNames)); } public static List<String> generateSchemas(List<String> sdNames) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java index 36a37d61b13..de68dbd7a11 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java @@ -19,7 +19,8 @@ public class CloudAccount implements Comparable<CloudAccount> { private static final Map<String, CloudMeta> META_BY_CLOUD = Map.of( "aws", new CloudMeta("Account ID", Pattern.compile("[0-9]{12}")), "azure", new CloudMeta("Subscription ID", Pattern.compile("[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}")), - "gcp", new CloudMeta("Project ID", Pattern.compile("[a-z][a-z0-9-]{4,28}[a-z0-9]"))); + "gcp", new CloudMeta("Project ID", Pattern.compile("[a-z][a-z0-9-]{4,28}[a-z0-9]")), + "yahoo", new CloudMeta("OpenStack Project", Pattern.compile("[a-zA-Z0-9._-]+"))); /** Empty value. When this is used, either implicitly or explicitly, the zone will use its default account */ public static final CloudAccount empty = new CloudAccount("", CloudName.DEFAULT); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java index 4fe6f8453de..f79fee08bc4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostFilter.java @@ -4,7 +4,6 @@ package com.yahoo.config.provision; import com.yahoo.text.StringUtilities; import java.util.Collection; -import java.util.Collections; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -44,27 +43,27 @@ public class HostFilter { /** Returns a filter which matches all hosts */ public static HostFilter all() { - return new HostFilter(Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + return new HostFilter(Set.of(), Set.of(), Set.of(), Set.of()); } /** Returns a filter which matches a given host only */ public static HostFilter hostname(String hostname) { - return new HostFilter(Collections.singleton(hostname), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + return new HostFilter(Set.of(hostname), Set.of(), Set.of(), Set.of()); } /** Returns a filter which matches a given flavor only */ public static HostFilter flavor(String flavor) { - return new HostFilter(Collections.emptySet(), Collections.singleton(flavor), Collections.emptySet(), Collections.emptySet()); + return new HostFilter(Set.of(), Set.of(flavor), Set.of(), Set.of()); } /** Returns a filter which matches a given cluster type only */ public static HostFilter clusterType(ClusterSpec.Type clusterType) { - return new HostFilter(Collections.emptySet(), Collections.emptySet(), Collections.singleton(clusterType), Collections.emptySet()); + return new HostFilter(Set.of(), Set.of(), Set.of(clusterType), Set.of()); } /** Returns a filter which matches a given cluster id only */ public static HostFilter clusterId(ClusterSpec.Id clusterId) { - return new HostFilter(Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.singleton(clusterId)); + return new HostFilter(Set.of(), Set.of(), Set.of(), Set.of(clusterId)); } /** Returns a host filter from three optional conditions */ diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CloudAccountTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CloudAccountTest.java index 5af9cdb9263..68bde6ee471 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/CloudAccountTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CloudAccountTest.java @@ -73,7 +73,7 @@ class CloudAccountTest { assertInvalidAccount("aws:123", "Invalid cloud account 'aws:123': Account ID must match '[0-9]{12}'"); assertInvalidAccount("gcp:123", "Invalid cloud account 'gcp:123': Project ID must match '[a-z][a-z0-9-]{4,28}[a-z0-9]'"); assertInvalidAccount("$something", "Invalid cloud account '$something': Must be on format '<cloud-name>:<account>' or 'default'"); - assertInvalidAccount("unknown:account", "Invalid cloud account 'unknown:account': Cloud name must be one of: aws, azure, gcp"); + assertInvalidAccount("unknown:account", "Invalid cloud account 'unknown:account': Cloud name must be one of: aws, azure, gcp, yahoo"); } private static void assertInvalidAccount(String account, String message) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java index 87165c0afef..93995bace63 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java @@ -4,7 +4,7 @@ package com.yahoo.config.provision; import com.yahoo.component.Vtag; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -45,10 +45,10 @@ public class HostFilterTest { @Test void testMultiConditionFilter() { - HostFilter typeAndId = HostFilter.from(Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList(ClusterSpec.Type.content), - Collections.singletonList(ClusterSpec.Id.from("type1"))); + HostFilter typeAndId = HostFilter.from(List.of(), + List.of(), + List.of(ClusterSpec.Type.content), + List.of(ClusterSpec.Id.from("type1"))); assertFalse(typeAndId.matches("anyhost", "flavor", membership("content/anyType/0/0/stateful"))); assertFalse(typeAndId.matches("anyhost", "flavor", membership("container/type1/0/0"))); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 2034a537b7c..3ec968e545a 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -180,7 +181,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher { String ret; System.out.println(proxyServer.getMode()); if (proxyServer.getMode().requiresConfigSource()) { - proxyServer.updateSourceConnections(Arrays.asList(sources.split(","))); + proxyServer.updateSourceConnections(List.of(sources.split(","))); ret = "Updated config sources to: " + sources; } else { ret = "Cannot update sources when in '" + proxyServer.getMode().name() + "' mode"; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index e69d76ec124..a34ad69d22c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -56,7 +55,7 @@ class MemoryCacheConfigClient implements ConfigSourceClient { @Override public List<String> getSourceConnections() { - return Collections.singletonList("N/A"); + return List.of("N/A"); } @Override diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 8074c1bd702..bf84e02ec9d 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -16,6 +16,7 @@ import net.jpountz.xxhash.XXHashFactory; import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -94,6 +95,7 @@ class UrlDownloadRpcServer { } private static Downloader downloader(String url) { + Objects.requireNonNull(url, "url cannot be null"); URI uri = new URI(url); return switch (uri.getScheme()) { case "http", "https" -> new UrlDownloader(); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java index 220734f5cec..6d77c6325c2 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloader.java @@ -22,11 +22,13 @@ class UrlDownloader implements Downloader { private static final Logger log = Logger.getLogger(UrlDownloader.class.getName()); private static final String CONTENTS_FILE_NAME = "contents"; + private static final String USER_AGENT_MODEL_DOWNLOADER = "Vespa/8.x (model download - https://github.com/vespa-engine/vespa)"; @Override public Optional<File> downloadFile(String url, File downloadDir) throws IOException { long start = System.currentTimeMillis(); HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestProperty("User-Agent", USER_AGENT_MODEL_DOWNLOADER); if (connection.getResponseCode() != 200) throw new RuntimeException("Download of URL '" + url + "' failed, got response code " + connection.getResponseCode()); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java index aa063b1ea54..656187ea8e5 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigTester.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.config.protocol.Payload; import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.util.ConfigUtils; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -30,7 +29,7 @@ import static com.yahoo.vespa.config.PayloadChecksum.Type.XXHASH64; public class ConfigTester { private static final long defaultTimeout = 10000; - private static final List<String> defContent = Collections.singletonList("bar string"); + private static final List<String> defContent = List.of("bar string"); static RawConfig fooConfig; static RawConfig barConfig; diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index baafe2ae2dd..69f6e0e8dae 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -51,7 +50,7 @@ public class MockConfigSourceClient implements ConfigSourceClient{ @Override public List<String> getSourceConnections() { - return Collections.singletonList("N/A"); + return List.of("N/A"); } @Override diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 65737d116c5..3d42967802f 100755..100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -49,7 +48,7 @@ public class ConfigSourceSet implements ConfigSource { * @param addresses Connection endpoints on the format "tcp/host:port". */ public ConfigSourceSet(String[] addresses) { - this(Arrays.asList(addresses)); + this(List.of(addresses)); } /** diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index e7f3555166e..bbad22e52e8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -444,7 +444,7 @@ public class ConfigSubscriber implements AutoCloseable { hasNewConfig = nextConfig(false); } catch (Exception e) { - log.log(SEVERE, "Exception on receiving config. Ignoring this change.", e); + log.log(isClosed() ? FINE : WARNING, "Exception on receiving config. Ignoring this change.", e); } try { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java index d33e1b422e0..03f791e5536 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -14,7 +14,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.util.List; import java.util.jar.JarFile; import java.util.zip.ZipEntry; @@ -56,7 +56,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); T config; try { - ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n"))); + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(List.of(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n"))); config = payload.toInstance(configClass, key.getConfigId()); } catch (UnsupportedEncodingException e) { throw new IllegalStateException(e); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java index dd08c1fe29c..d4e4e075e23 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -8,7 +8,8 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.PayloadChecksums; -import java.util.Arrays; +import java.util.List; + /** * Subscription used when config id is raw:... @@ -34,7 +35,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc } if (payload == null) { payload = inputPayload; - ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); + ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(List.of(payload.split("\n"))); setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()), PayloadChecksums.empty()); return true; } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java index f640f5c1294..2545b82b3dd 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -567,7 +566,7 @@ public class ConfigDefinition { public void addEnumDef(String id, String valsCommaSep, String defVal) { String[] valArr = valsCommaSep.split(","); - addEnumDef(id, Arrays.asList(valArr), defVal); + addEnumDef(id, List.of(valArr), defVal); } public void addStringDef(String id, String defVal) { diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java index c8e86c97661..8e8a48580d8 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java @@ -4,7 +4,8 @@ package com.yahoo.vespa.config; import com.yahoo.config.codegen.CNode; import com.yahoo.config.codegen.LeafCNode; -import java.util.Arrays; +import java.util.List; + /** * Builds a ConfigDefinition from a tree of CNodes. @@ -183,9 +184,9 @@ public class ConfigDefinitionBuilder { private static void addNode(ConfigDefinition def, LeafCNode.EnumLeaf leaf) { if (leaf.getDefaultValue() != null) { - def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), leaf.getDefaultValue().getValue()); + def.addEnumDef(leaf.getName(), List.of(leaf.getLegalValues()), leaf.getDefaultValue().getValue()); } else { - def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), null); + def.addEnumDef(leaf.getName(), List.of(leaf.getLegalValues()), null); } } diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java index 9964e63bfd6..3f7d9c15fa7 100755..100644 --- a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.protocol.Payload; import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.util.ConfigUtils; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -40,7 +39,7 @@ public class RawConfig extends ConfigInstance { * @param defMd5 The md5 sum of the .def-file. */ public RawConfig(ConfigKey<?> key, String defMd5) { - this(key, defMd5, null, PayloadChecksums.empty(), 0L, false, 0, Collections.emptyList(), Optional.empty()); + this(key, defMd5, null, PayloadChecksums.empty(), 0L, false, 0, List.of(), Optional.empty()); } public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, PayloadChecksums payloadChecksums, long generation, diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index 6a0f8ea1cdc..4c61c995491 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -28,7 +28,6 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -139,7 +138,7 @@ public class LoadTester { if (!name.endsWith(".def")) continue; String contents = IOUtils.readFile(f); ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefFile(f); - ret.put(key, new Tuple2<>(ConfigUtils.getDefMd5(Arrays.asList(contents.split("\n"))), contents.split("\n"))); + ret.put(key, new Tuple2<>(ConfigUtils.getDefMd5(List.of(contents.split("\n"))), contents.split("\n"))); } System.out.println("# Read " + ret.size() + " def files from " + defDir.getPath()); return ret; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java index 6b3c1f0040d..a4c4dfde8f0 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java @@ -7,7 +7,6 @@ import com.yahoo.slime.*; import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -47,7 +46,7 @@ public class DefContent { } public static DefContent fromArray(String[] schema) { - return fromList(Arrays.asList(schema)); + return fromList(List.of(schema)); } /** diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java index 851ef28155d..29582fedc44 100644 --- a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java @@ -8,7 +8,6 @@ import com.yahoo.foo.SimpletypesConfig; import com.yahoo.vespa.config.ConfigPayload; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals; @@ -216,7 +215,7 @@ public class CfgConfigPayloadBuilderTest { @Test public void test_config_with_comments() { CfgConfigPayloadBuilderTest.assertDeserializedConfigEqualsJson( - Arrays.asList( + List.of( "fielda b\n", "#fielda c\n", "#fieldb c\n", @@ -230,7 +229,7 @@ public class CfgConfigPayloadBuilderTest { @Test public void testConvertingMaps() { - List<String> payload = Arrays.asList( + List<String> payload = List.of( "intmap{\"foo\"} 1337", "complexmap{\"key\"}.foo 1337", "nestedmap{\"key1\"}.foo{\"key2\"}.bar 1337" diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java index d737a2d5930..3327e94c32b 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.ConfigTransformer; import org.junit.Test; import java.io.File; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -55,7 +54,7 @@ public class ConfigInstancePayloadTest { doublearr(123.0). stringarr("bar"). enumarr(Enumarr.VALUES). - refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + refarr(List.of(":parent:", ":parent", "parent:")). // test collection based setter fileArr("bin"). pathArr(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))). intMap("one", 1). diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java index 1c6ca46eb04..bbd4ae7e797 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java @@ -49,15 +49,15 @@ public class ConfigInstanceTest { @Ignore public void testRetainOldValuesOnConfigUpdates() { AppConfig config = new AppConfig(new AppConfig.Builder()); - //config.setConfig(Arrays.asList("message \"one\"", "times 333"), "", 0L); + //config.setConfig(List.of("message \"one\"", "times 333"), "", 0L); assertEquals("one", config.message()); assertEquals(333, config.times()); - //config.setConfig(Arrays.asList("message \"two\""), "", 0L); + //config.setConfig(List.of("message \"two\""), "", 0L); assertEquals("two", config.message()); assertEquals("config.times retains previously set value", 333, config.times()); - //config.setConfig(Arrays.asList("times 666"), "", 0L); + //config.setConfig(List.of("times 666"), "", 0L); assertEquals("config.message retains previously set value", "two", config.message()); assertEquals(666, config.times()); } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java index 58c8a740bdb..6b392db5906 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import org.junit.Test; import java.io.File; -import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; import static com.yahoo.foo.FunctionTestConfig.*; @@ -97,7 +97,7 @@ public class ConfigInstanceUtilTest { doublearr(123.0). stringarr("bar"). enumarr(Enumarr.VALUES). - refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + refarr(List.of(":parent:", ":parent", "parent:")). // test collection based setter fileArr("bin"). basicStruct(b -> b. diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java index 69ec2ff2d7b..c347880f239 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; import org.junit.Test; -import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; @@ -78,7 +77,7 @@ public class ConfigSubscriptionTest { @Test public void testSubscribeWithException() { TestConfigSubscriber sub = new TestConfigSubscriber(); - ConfigSourceSet configSourceSet = new ConfigSourceSet(Collections.singletonList("tcp/localhost:99999")); + ConfigSourceSet configSourceSet = new ConfigSourceSet(List.of("tcp/localhost:99999")); try { sub.subscribe(SimpletypesConfig.class, "configid", configSourceSet, new TimingValues().setSubscribeTimeout(100)); fail(); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java index ff2b2bbaf5f..021178d88b1 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java @@ -8,7 +8,8 @@ import com.yahoo.foo.StructtypesConfig; import com.yahoo.foo.MaptypesConfig; import org.junit.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.foo.MaptypesConfig.Innermap; import static org.hamcrest.CoreMatchers.is; @@ -44,7 +45,7 @@ public class ConfigBuilderMergeTest { StructtypesConfig.Simple.Builder simpleBuilder = new StructtypesConfig.Simple.Builder(); simpleBuilder.name(name); simpleBuilder.gender(StructtypesConfig.Simple.Gender.Enum.valueOf(gender)); - simpleBuilder.emails(Arrays.asList(emails)); + simpleBuilder.emails(List.of(emails)); builder.simple(simpleBuilder); return builder; } diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java index 9223c39055b..06e3297d2fe 100644 --- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.Test; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -29,7 +28,7 @@ import static org.junit.Assert.assertNull; public class RawConfigTest { private static final ConfigKey<?> key = new ConfigKey<>("foo", "id", "bar"); - private static final List<String> defContent = Arrays.asList("version=1", "anInt int"); + private static final List<String> defContent = List.of("version=1", "anInt int"); private static final String defMd5 = ConfigUtils.getDefMd5FromRequest("", defContent); private static final PayloadChecksums payloadChecksums = PayloadChecksums.from("012345", ""); private static final Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed()); diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index b80a7ac73e3..81e587fcace 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -36,6 +36,8 @@ vespa_generate_config(configdefinitions load-type.def) install_config_definition(load-type.def vespa.config.content.load-type.def) vespa_generate_config(configdefinitions logforwarder.def) install_config_definition(logforwarder.def cloud.config.logforwarder.def) +vespa_generate_config(configdefinitions open-telemetry.def) +install_config_definition(open-telemetry.def cloud.config.open-telemetry.def) vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) install_config_definition(messagetyperouteselectorpolicy.def vespa.config.content.messagetyperouteselectorpolicy.def) vespa_generate_config(configdefinitions model.def) diff --git a/configdefinitions/src/vespa/open-telemetry.def b/configdefinitions/src/vespa/open-telemetry.def new file mode 100644 index 00000000000..3b379f6ca16 --- /dev/null +++ b/configdefinitions/src/vespa/open-telemetry.def @@ -0,0 +1,8 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# For now - store entire config in one string +yaml string + +# Referenced paths +refPaths[] string diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index de67d4336e9..a5d86cc91ba 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -29,7 +29,7 @@ response_sequencer_type enum {LATENCY, THROUGHPUT, ADAPTIVE} default=ADAPTIVE re ## Should follow stor-distributormanager:splitsize (16MB). bucket_merge_chunk_size int default=16772216 restart -## Whether or not to use async message handling when scheduling storage messages from FileStorManager. +## Whether to use async message handling when scheduling storage messages from FileStorManager. ## ## When turned on, the calling thread (e.g. FNET network thread when using Storage API RPC) ## gets the next async message to handle (if any) as part of scheduling a storage message. @@ -61,3 +61,10 @@ async_operation_throttler.window_size_backoff double default=0.95 async_operation_throttler.min_window_size int default=20 async_operation_throttler.max_window_size int default=-1 # < 0 implies INT_MAX async_operation_throttler.resize_rate double default=3.0 + +## Maximum number of enqueued put/remove/update operations towards a given bucket +## that can be dispatched asynchronously as a batch under the same write lock. +## This prevents pipeline stalls when many write operations are in-flight to the +## same bucket, as each operation would otherwise have to wait for the completion +## of all prior writes to the bucket. +max_feed_op_batch_size int default=1 diff --git a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java index a3a5727e30c..fe6e328020f 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/BuilderGenerator.java @@ -99,7 +99,7 @@ public class BuilderGenerator { } String uninitializedList = (scalarsWithoutDefault.size() > 0) - ? "Arrays.asList(\n" + indentCode(INDENTATION, String.join(",\n", scalarsWithoutDefault) + "\n)") + ? "List.of(\n" + indentCode(INDENTATION, String.join(",\n", scalarsWithoutDefault) + "\n)") : ""; return "private Set<String> " + INTERNAL_PREFIX + "uninitialized = new HashSet<String>(" + uninitializedList + ");"; diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference index b17655e317d..d068853b167 100644 --- a/configgen/src/test/resources/allfeatures.reference +++ b/configgen/src/test/resources/allfeatures.reference @@ -101,7 +101,7 @@ public final class AllfeaturesConfig extends ConfigInstance { } public static final class Builder implements ConfigInstance.Builder { - private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + private Set<String> __uninitialized = new HashSet<String>(List.of( "boolVal", "intVal", "longVal", @@ -1730,7 +1730,7 @@ public final class AllfeaturesConfig extends ConfigInstance { public final static class MyArray extends InnerNode { public static final class Builder implements ConfigBuilder { - private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + private Set<String> __uninitialized = new HashSet<String>(List.of( "refVal" )); @@ -2040,7 +2040,7 @@ public final class AllfeaturesConfig extends ConfigInstance { public final static class MyMap extends InnerNode { public static final class Builder implements ConfigBuilder { - private Set<String> __uninitialized = new HashSet<String>(Arrays.asList( + private Set<String> __uninitialized = new HashSet<String>(List.of( "refVal" )); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java index 51918b4902a..bfad5af504c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; -import java.util.Collections; import java.util.Map; /** @@ -24,7 +23,7 @@ public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { this.repo = new ConfigDefinitionRepo() { @Override public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { - return Collections.emptyMap(); + return Map.of(); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index 3de35a20008..57ecd175435 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.config.server.http.JSONResponse; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -75,7 +74,7 @@ public class FileDistributionStatus extends AbstractComponent { if (request.isError()) { return new HostStatus(hostname, Status.UNKNOWN, - Collections.emptyMap(), + Map.of(), "error: " + request.errorMessage() + "(" + request.errorCode() + ")"); } else { Map<String, Double> fileReferenceStatuses = new HashMap<>(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 26732d2e20f..7189ae12c66 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -206,6 +206,8 @@ public class ModelContextImpl implements ModelContext { private final int contentLayerMetadataFeatureLevel; private final String unknownConfigDefinition; private final int searchHandlerThreadpool; + private final int persistenceThreadMaxFeedOpBatchSize; + private final boolean logserverOtelCol; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -248,6 +250,8 @@ public class ModelContextImpl implements ModelContext { this.searchHandlerThreadpool = flagValue(source, appId, version, Flags.SEARCH_HANDLER_THREADPOOL); this.alwaysMarkPhraseExpensive = flagValue(source, appId, version, Flags.ALWAYS_MARK_PHRASE_EXPENSIVE); this.sortBlueprintsByCost = flagValue(source, appId, version, Flags.SORT_BLUEPRINTS_BY_COST); + this.persistenceThreadMaxFeedOpBatchSize = flagValue(source, appId, version, Flags.PERSISTENCE_THREAD_MAX_FEED_OP_BATCH_SIZE); + this.logserverOtelCol = flagValue(source, appId, version, Flags.LOGSERVER_OTELCOL_AGENT); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -298,6 +302,8 @@ public class ModelContextImpl implements ModelContext { @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } @Override public int searchHandlerThreadpool() { return searchHandlerThreadpool; } @Override public boolean sortBlueprintsByCost() { return sortBlueprintsByCost; } + @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } + @Override public boolean logserverOtelCol() { return logserverOtelCol; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index f0228f40ad2..a634519dd0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -24,7 +24,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; @@ -109,7 +108,7 @@ public class ZooKeeperDeployer { void initialize() { curator.create(sessionPath); - for (String subPath : Arrays.asList(DEFCONFIGS_ZK_SUBPATH, + for (String subPath : List.of(DEFCONFIGS_ZK_SUBPATH, USER_DEFCONFIGS_ZK_SUBPATH, USERAPP_ZK_SUBPATH, ZKApplicationPackage.fileRegistryNode)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java index 06e5ded4cdb..c80d034a2f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -19,7 +19,7 @@ import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests; import com.yahoo.vespa.config.server.http.v2.request.TenantRequest; import com.yahoo.vespa.config.util.ConfigUtils; -import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -190,7 +190,7 @@ public class HttpConfigRequest implements GetConfigRequest, TenantRequest { @Override public DefContent getDefContent() { - return DefContent.fromList(Collections.emptyList()); + return DefContent.fromList(List.of()); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index b0ecedd4853..6d671377e63 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -48,7 +48,6 @@ import com.yahoo.vespa.filedistribution.FileReferenceDownload; import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -131,7 +130,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList this.superModelRequestHandler = superModelRequestHandler; metricUpdaterFactory = metrics; supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); - this.metrics = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); + this.metrics = metrics.getOrCreateMetricUpdater(Map.of()); BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); int rpcWorkerThreads = (config.numRpcThreads() == 0) ? threadsToUse() : config.numRpcThreads(); executorService = new ThreadPoolExecutor(rpcWorkerThreads, rpcWorkerThreads, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 546277c4aba..ac483ea20c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -64,7 +64,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -851,7 +850,7 @@ public class SessionRepository { File[] sdFiles = sdDir.listFiles(); if (sdFiles != null) { Files.createDirectories(schemasDir.toPath()); - Arrays.asList(sdFiles).forEach(file -> Exceptions.uncheck( + List.of(sdFiles).forEach(file -> Exceptions.uncheck( () -> Files.move(file.toPath(), schemasDir.toPath().resolve(file.toPath().getFileName()), StandardCopyOption.REPLACE_EXISTING))); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 8c4445f897c..f9f477c9693 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -194,7 +194,7 @@ public class TenantRepository { this.configserverConfig = configserverConfig; this.curator = curator; this.metrics = metrics; - metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); + metricUpdater = metrics.getOrCreateMetricUpdater(Map.of()); this.zkCacheExecutor = zkCacheExecutor; this.zkApplicationWatcherExecutor = zkApplicationWatcherExecutor; this.zkSessionWatcherExecutor = zkSessionWatcherExecutor; @@ -312,7 +312,7 @@ public class TenantRepository { } } - if (failed.size() > 0) + if (!failed.isEmpty()) throw new RuntimeException("Could not create all tenants when bootstrapping, failed to create: " + failed); metricUpdater.setTenants(tenants.size()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 801f599c2d8..34900f61463 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.BooleanSupplier; @@ -311,7 +310,7 @@ public class ConfigServerBootstrapTest { } private Host createHost(String hostname, String version) { - return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); + return new Host(hostname, List.of(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); } private VipStatus createVipStatus(StateMonitor stateMonitor) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java index 4d23c4c6ffa..64417536f8b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java @@ -56,7 +56,7 @@ public class ModelFactoryRegistryTest { TestFactory b = new TestFactory(new Version(5, 58, 1)); TestFactory c = new TestFactory(new Version(5, 48, 44)); TestFactory d = new TestFactory(new Version(5, 18, 44)); - ModelFactoryRegistry registry = new ModelFactoryRegistry(Arrays.asList(a, b, c, d)); + ModelFactoryRegistry registry = new ModelFactoryRegistry(List.of(a, b, c, d)); assertEquals(4, registry.getFactories().size()); assertTrue(registry.getFactories().contains(a)); assertTrue(registry.getFactories().contains(b)); @@ -70,23 +70,9 @@ public class ModelFactoryRegistryTest { registry.getFactory(new Version(3, 2, 1)); } - private static class TestFactory implements ModelFactory { + private record TestFactory(Version version) implements ModelFactory { - private final Version version; - - TestFactory(Version version) { - this.version = version; - } - - @Override - public Version version() { - return version; - } - - @Override - public Model createModel(ModelContext modelContext) { - return null; - } + @Override public Model createModel(ModelContext modelContext) { return null; } @Override public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index af86463bb81..39f1cb70112 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -30,8 +30,8 @@ import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; -import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -73,7 +73,7 @@ public class SuperModelControllerTest { public void test_unknown_config_definition() { PayloadChecksums payloadChecksums = PayloadChecksums.empty(); Request request = createWithParams(new ConfigKey<>("foo", "id", "bar", null), - DefContent.fromList(Collections.emptyList()), "fromHost", + DefContent.fromList(List.of()), "fromHost", payloadChecksums, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()) .getRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 28c95ba33c8..74b08b83791 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -21,7 +21,6 @@ import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -88,7 +87,7 @@ public class SuperModelRequestHandlerTest { assertTrue(controller.getHandler().getSuperModel().applicationModels().keySet().containsAll(List.of(foo, bar, baz))); controller.removeApplication(bar); assertEquals(2, controller.getHandler().getSuperModel().applicationModels().size()); - assertEquals(Arrays.asList(foo, baz), new ArrayList<>(controller.getHandler().getSuperModel().applicationModels().keySet())); + assertEquals(List.of(foo, baz), new ArrayList<>(controller.getHandler().getSuperModel().applicationModels().keySet())); assertEquals(gen + 5, controller.getHandler().getGeneration()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java index ddcee892594..28090b42e75 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.application; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -52,7 +51,7 @@ public class ApplicationMapperTest { @Test (expected = VersionDoesNotExistException.class) public void testGetForVersionThrows() { - applicationMapper.register(appId, ApplicationVersions.fromList(Arrays.asList(applications.get(0), applications.get(2)))); + applicationMapper.register(appId, ApplicationVersions.fromList(List.of(applications.get(0), applications.get(2)))); applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(1)), Instant.now()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java index 9345b45bab4..23a425b99e2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.application; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -46,7 +45,7 @@ public class ApplicationVersionsTest { @Test (expected = VersionDoesNotExistException.class) public void testGetForVersionOrLatestThrows() { - applicationVersions = ApplicationVersions.fromList(Arrays.asList(applications.get(0), applications.get(2))); + applicationVersions = ApplicationVersions.fromList(List.of(applications.get(0), applications.get(2))); applicationVersions.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 93c41f1b087..b735c69f247 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -18,7 +18,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.zip.GZIPOutputStream; @@ -104,7 +103,7 @@ public class CompressedApplicationInputStreamTest { try (CompressedApplicationInputStream unpacked = streamFromTarGz(gzFile)) { outApp = unpacked.decompress(); } - List<File> files = Arrays.asList(Objects.requireNonNull(outApp.listFiles())); + List<File> files = List.of(Objects.requireNonNull(outApp.listFiles())); assertEquals(5, files.size()); assertTrue(files.contains(new File(outApp, "services.xml"))); assertTrue(files.contains(new File(outApp, "hosts.xml"))); @@ -121,13 +120,13 @@ public class CompressedApplicationInputStreamTest { assertEquals(1, ext.listFiles().length); assertEquals(new File(ext, "foo").getAbsolutePath(), ext.listFiles()[0].getAbsolutePath()); - files = Arrays.asList(ext.listFiles()); + files = List.of(ext.listFiles()); File foo = files.get(files.indexOf(new File(ext, "foo"))); assertTrue(foo.isDirectory()); assertEquals(1, foo.listFiles().length); assertEquals(new File(foo, "bar").getAbsolutePath(), foo.listFiles()[0].getAbsolutePath()); - files = Arrays.asList(foo.listFiles()); + files = List.of(foo.listFiles()); File bar = files.get(files.indexOf(new File(foo, "bar"))); assertTrue(bar.isDirectory()); assertEquals(1, bar.listFiles().length); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 0fe39fd0224..a5ce29b6494 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -17,7 +17,6 @@ import org.junit.rules.TemporaryFolder; import java.net.URI; import java.time.Duration; -import java.util.Arrays; import java.util.List; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; @@ -101,7 +100,7 @@ public class ConfigConvergenceCheckerTest { } { // Model with two hosts on different generations - MockModel model = new MockModel(Arrays.asList( + MockModel model = new MockModel(List.of( MockModel.createContainerHost(service.getHost(), service.getPort()), MockModel.createContainerHost(service2.getHost(), service2.getPort())) ); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java index 4a2f422252b..30e1373b7f6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/FileDistributionStatusTest.java @@ -17,7 +17,6 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -119,7 +118,7 @@ public class FileDistributionStatusTest { fileReferenceStatuses2.put("1234", 1.0); HostStatus localhost2 = statusFinished("localhost2", Status.FINISHED, fileReferenceStatuses2); - FileDistributionStatus status = new MockStatus(new HashSet<>(Arrays.asList(localhost, localhost2))); + FileDistributionStatus status = new MockStatus(new HashSet<>(List.of(localhost, localhost2))); application = createApplication("localhost", "localhost2"); HttpResponse response = getStatus(status, application); assertResponse(200, @@ -156,7 +155,7 @@ public class FileDistributionStatusTest { } private Application createApplication(String... hostname) { - return createApplication(Arrays.asList(hostname)); + return createApplication(List.of(hostname)); } private Application createApplication(List<String> hostnames) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index 1a6669e93b2..f57148dd8d0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -21,8 +21,9 @@ import org.mockito.ArgumentCaptor; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; -import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo; @@ -119,9 +120,9 @@ public class HttpProxyTest { "state http external query"); ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", ClusterSpec.Type.content, 1234, "rpc"); - HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); + HostInfo hostInfo = new HostInfo(hostname, List.of(container, serviceNoStatePort)); - return new MockModel(Collections.singleton(hostInfo)); + return new MockModel(Set.of(hostInfo)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index c18dad1ea67..4ba507bf5d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -40,7 +40,7 @@ public class MockModel implements Model { ClusterSpec.Type.container, statePort, "state"); ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "logserver", "logserver", ClusterSpec.Type.admin, 1234, "logtp"); - return new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); + return new HostInfo(hostname, List.of(container, serviceNoStatePort)); } static MockModel createConfigProxies(List<String> hostnames, int rpcPort) { @@ -48,7 +48,7 @@ public class MockModel implements Model { hostnames.forEach(hostname -> { ServiceInfo configProxy = createServiceInfo(hostname, "configproxy", "configproxy", ClusterSpec.Type.admin, rpcPort, "rpc"); - hostInfos.add(new HostInfo(hostname, Collections.singletonList(configProxy))); + hostInfos.add(new HostInfo(hostname, List.of(configProxy))); }); return new MockModel(hostInfos); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 14d49f9d9d6..a613dc91211 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -42,7 +42,6 @@ import java.io.File; import java.io.IOException; import java.time.Clock; import java.time.Duration; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.Set; @@ -306,7 +305,7 @@ public class TenantApplicationsTest { } private ModelFactoryRegistry createRegistry() { - return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), + return new ModelFactoryRegistry(List.of(new TestModelFactory(vespaVersion), new TestModelFactory(new Version(3, 2, 1)))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 67a1a335067..68add64ddd9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -53,7 +53,6 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -219,7 +218,7 @@ public class DeployTester { public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { if ( ! validationParameters.ignoreValidationErrors()) throw new IllegalArgumentException("Model building fails"); - return new ModelCreateResult(createModel(modelContext), Collections.emptyList()); + return new ModelCreateResult(createModel(modelContext), List.of()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 838b1b6b209..512f4dff6b7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -47,7 +47,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -581,7 +580,7 @@ public class HostedDeployTest { } private Host createHost(String hostname, String version) { - return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.ofNullable(version).map(Version::fromString)); + return new Host(hostname, List.of(), Optional.empty(), Optional.ofNullable(version).map(Version::fromString)); } private DeployTester createTester(List<Host> hosts, List<ModelFactory> modelFactories, Zone zone) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java index 13192499603..ee694c984fb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java @@ -8,8 +8,8 @@ import com.yahoo.jdisc.http.HttpRequest; import org.junit.Test; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; @@ -92,11 +92,7 @@ public abstract class ContentHandlerTestBase extends SessionHandlerTest { protected abstract HttpResponse doRequest(HttpRequest.Method method, String path); private String generateResultArray(String... files) { - Collection<String> output = Collections2.transform(Arrays.asList(files), input -> "\"" + baseUrl + input + "\""); - StringBuilder sb = new StringBuilder(); - sb.append("["); - sb.append(Joiner.on(",").join(output)); - sb.append("]"); - return sb.toString(); + Collection<String> output = Collections2.transform(List.of(files), input -> "\"" + baseUrl + input + "\""); + return "[" + Joiner.on(",").join(output) + "]"; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 34045cf3804..452ee53c439 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -17,7 +17,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; -import java.util.Collections; +import java.util.Map; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST; @@ -90,7 +90,7 @@ public class HttpGetConfigHandlerTest { @Test public void require_that_nocache_property_works() throws IOException { - HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); + HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Map.of("nocache", "true")); HttpResponse response = handler.handle(request); String renderedString = SessionHandlerTest.getRenderedString(response); assertTrue(renderedString, renderedString.startsWith(expected)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java index a7f86133fce..b58bca074b8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java @@ -11,7 +11,7 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; -import java.util.Collections; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -39,7 +39,7 @@ public class HttpHandlerTest { HttpRequest.createTestRequest("foo", com.yahoo.jdisc.http.HttpRequest.Method.GET, null, - Collections.singletonMap("timeout", "1.5")), Duration.ofSeconds(5)).toMillis()); + Map.of("timeout", "1.5")), Duration.ofSeconds(5)).toMillis()); } private static class HttpTestHandler extends HttpHandler { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 6f2c7366b62..4c140159e12 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -24,7 +24,6 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; -import java.util.Collections; import java.util.Map; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; @@ -126,7 +125,7 @@ public class HttpGetConfigHandlerTest { @Test public void require_that_nocache_property_works() throws IOException { - HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); + HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Map.of("nocache", "true")); HttpResponse response = handler.handle(request); String renderedString = SessionHandlerTest.getRenderedString(response); assertTrue(renderedString, renderedString.startsWith(expected)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index b8c9a2f8fb8..4fb514bde17 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -30,7 +30,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -93,7 +92,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Ignore @Test public void require_that_from_parameter_cannot_be_set_if_data_in_request() throws IOException { - HttpRequest request = post(Collections.singletonMap("from", "active")); + HttpRequest request = post(Map.of("from", "active")); HttpResponse response = createHandler().handle(request); assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.ErrorCode.BAD_REQUEST, "Parameter 'from' is illegal for POST"); } @@ -114,14 +113,14 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { private void assertIllegalFromParameter(String fromValue) throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); - HttpRequest request = post(outFile, postHeaders, Collections.singletonMap("from", fromValue)); + HttpRequest request = post(outFile, postHeaders, Map.of("from", fromValue)); assertHttpStatusCodeErrorCodeAndMessage(createHandler().handle(request), BAD_REQUEST, HttpErrorResponse.ErrorCode.BAD_REQUEST, "Parameter 'from' has illegal value '" + fromValue + "'"); } @Test public void require_that_prepare_url_is_returned_on_success() throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(temporaryFolder.getRoot().toPath()); - Map<String, String> parameters = Collections.singletonMap("name", "foo"); + Map<String, String> parameters = Map.of("name", "foo"); HttpResponse response = createHandler().handle(post(outFile, postHeaders, parameters)); assertNotNull(response); assertEquals(OK, response.getStatus()); @@ -160,7 +159,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Test public void require_that_application_urls_can_be_given_as_from_parameter() throws Exception { ApplicationId applicationId = ApplicationId.from(tenant.value(), "foo", "quux"); - HttpRequest request = post(Collections.singletonMap( + HttpRequest request = post(Map.of( "from", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux")); assertEquals(applicationId, SessionCreateHandler.getFromApplicationId(request)); @@ -180,7 +179,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { public void require_that_content_type_is_parsed_correctly() throws FileNotFoundException { HttpRequest request = post(new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)), Map.of("Content-Type", "multipart/form-data; charset=ISO-8859-1; boundary=g5gJAzUWl_t6"), - Collections.emptyMap()); + Map.of()); // Valid header should validate ok SessionCreateHandler.validateDataAndHeader(request, List.of(ContentType.MULTIPART_FORM_DATA.getMimeType())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java index 2ca0155ac6f..0773ace0051 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponseTest.java @@ -17,7 +17,6 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -80,7 +79,7 @@ public class DelayedConfigResponseTest { private JRTServerConfigRequest createRequest(String configName, String configId, long generation, long timeout, String namespace) { Request request = createWithParams(new ConfigKey<>(configName, configId, namespace, null), - DefContent.fromList(Collections.emptyList()), "fromHost", + DefContent.fromList(List.of()), "fromHost", PayloadChecksums.empty(), generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()) .getRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index f736eab0576..0ec651468cd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -61,7 +61,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -192,7 +191,7 @@ public class SessionPreparerTest { public void require_exception_for_overlapping_host() throws IOException { FilesApplicationPackage app = getApplicationPackage(testApp); HostRegistry hostValidator = new HostRegistry(); - hostValidator.update(applicationId("foo"), Collections.singletonList("mytesthost")); + hostValidator.update(applicationId("foo"), List.of("mytesthost")); preparer.prepare(hostValidator, new BaseDeployLogger(), new PrepareParams.Builder().applicationId(applicationId("default")).build(), Optional.empty(), Instant.now(), app.getAppDir(), app, createSessionZooKeeperClient()); } @@ -206,7 +205,7 @@ public class SessionPreparerTest { FilesApplicationPackage app = getApplicationPackage(testApp); HostRegistry hostValidator = new HostRegistry(); ApplicationId applicationId = applicationId(); - hostValidator.update(applicationId, Collections.singletonList("mytesthost")); + hostValidator.update(applicationId, List.of("mytesthost")); preparer.prepare(hostValidator, logger, new PrepareParams.Builder().applicationId(applicationId).build(), Optional.empty(), Instant.now(), app.getAppDir(), app, createSessionZooKeeperClient()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 515ecf0005e..9ef1f42f247 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -46,7 +46,6 @@ import org.mockito.Mock; import org.xml.sax.SAXException; import java.io.IOException; import java.time.Clock; -import java.util.Arrays; import java.util.List; import java.util.Set; @@ -174,7 +173,7 @@ public class TenantRepositoryTest { @Test public void testTenantWatching() throws Exception { TenantName newTenant = TenantName.from("newTenant"); - List<TenantName> expectedTenants = Arrays.asList(TenantName.defaultName(), newTenant); + List<TenantName> expectedTenants = List.of(TenantName.defaultName(), newTenant); try { tenantRepository.addTenant(newTenant); // Poll for the watcher to pick up the tenant from zk, and add it diff --git a/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java b/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java index 12a833881f7..2c2ee5226a6 100644 --- a/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java +++ b/container-core/src/main/java/com/yahoo/component/chain/ChainedComponent.java @@ -11,9 +11,7 @@ import com.yahoo.component.chain.dependencies.Provides; import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; /** @@ -70,7 +68,7 @@ public abstract class ChainedComponent extends AbstractComponent { // TODO: move to vespajlib. private static List<String> allOf(List<String> symbols, String... otherSymbols) { List<String> result = new ArrayList<>(symbols); - result.addAll(Arrays.asList(otherSymbols)); + result.addAll(List.of(otherSymbols)); return result; } @@ -88,9 +86,9 @@ public abstract class ChainedComponent extends AbstractComponent { Annotation annotation = component.getClass().getAnnotation(annotationClass); if (annotation != null) { Object values = annotationClass.getMethod("value").invoke(annotation); - return Arrays.asList((String[])values); + return List.of((String[])values); } - return Collections.emptyList(); + return List.of(); } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { throw new RuntimeException(e); diff --git a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java index fef23f3cc27..c0bc01be27c 100644 --- a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java +++ b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.component.chain.dependencies.ordering; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; @@ -50,7 +49,7 @@ public class ChainBuilder<T extends ChainedComponent> { } private Set<String> set(String... s) { - return new HashSet<>(Arrays.asList(s)); + return new HashSet<>(List.of(s)); } public PhaseNameProvider addPhase(Phase phase) { diff --git a/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java index bd35d257813..b49f519906f 100644 --- a/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java +++ b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java @@ -15,6 +15,7 @@ import com.yahoo.component.ComponentSpecification; public final class BundleInstantiationSpecification { public static final String CONTAINER_SEARCH_AND_DOCPROC = "container-search-and-docproc"; + public static final String MODEL_INTEGRATION = "model-integration"; public final ComponentId id; public final ComponentSpecification classId; diff --git a/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java b/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java index da5b040e296..b00455b1e2a 100644 --- a/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java +++ b/container-core/src/main/java/com/yahoo/container/bundle/MockBundle.java @@ -157,7 +157,7 @@ public class MockBundle implements Bundle, BundleWiring { @Override public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) { - return Collections.emptyMap(); + return Map.of(); } @SuppressWarnings("unchecked") @@ -239,7 +239,7 @@ public class MockBundle implements Bundle, BundleWiring { @Override public Collection<String> listResources(String p1, String p2, int p3) { - return Collections.emptyList(); + return List.of(); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java index fcccc02d143..5a04bd8eae1 100644 --- a/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-core/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -8,7 +8,6 @@ import com.yahoo.container.di.config.Subscriber; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.vespa.config.ConfigKey; -import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -135,7 +134,7 @@ public final class ConfigRetriever { private void resetComponentSubscriberIfBootstrap(ConfigSnapshot configSnapshot) { if (configSnapshot instanceof BootstrapConfigs) { - setupComponentSubscriber(Collections.emptySet()); + setupComponentSubscriber(Set.of()); } } diff --git a/container-core/src/main/java/com/yahoo/container/di/Osgi.java b/container-core/src/main/java/com/yahoo/container/di/Osgi.java index 0ff640031fc..7bab8605a3e 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Osgi.java +++ b/container-core/src/main/java/com/yahoo/container/di/Osgi.java @@ -10,8 +10,6 @@ import org.osgi.framework.Bundle; import java.util.Collection; import java.util.Set; -import static java.util.Collections.emptySet; - /** * This interface has default implementations of all methods, to allow using it * for testing, instead of mocking or a test implementation. @@ -49,7 +47,7 @@ public interface Osgi { * @return The set of bundles that are no longer needed by the new or latest good generation. */ default Set<Bundle> completeBundleGeneration(GenerationStatus status) { - return emptySet(); + return Set.of(); } default Class<?> resolveClass(BundleInstantiationSpecification spec) { diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index 98b4d0f4d63..5db57250bdf 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -23,7 +23,6 @@ import java.lang.reflect.Type; import java.lang.reflect.TypeVariable; import java.lang.reflect.WildcardType; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -369,7 +368,7 @@ public class ComponentGraph { public static boolean isBindingAnnotation(Annotation annotation) { LinkedList<Class<?>> queue = new LinkedList<>(); queue.add(annotation.getClass()); - queue.addAll(Arrays.asList(annotation.getClass().getInterfaces())); + queue.addAll(List.of(annotation.getClass().getInterfaces())); while (!queue.isEmpty()) { Class<?> clazz = queue.removeFirst(); diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java index 4500a5636f4..dcc024ef2b1 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -199,7 +199,7 @@ public class ComponentNode extends Node { @Override public boolean equals(Object other) { if (other instanceof ComponentNode that) { - return super.equals(that) && equalEdges(Arrays.asList(this.arguments), Arrays.asList(that.arguments)) && this.usedConfigs().equals(that.usedConfigs()); + return super.equals(that) && equalEdges(List.of(this.arguments), List.of(that.arguments)) && this.usedConfigs().equals(that.usedConfigs()); } else { return false; } @@ -225,7 +225,7 @@ public class ComponentNode extends Node { List<Pair<Type, List<Annotation>>> ret = new ArrayList<>(); for (int i = 0; i < types.length; i++) { - ret.add(new Pair<>(types[i], Arrays.asList(annotations[i]))); + ret.add(new Pair<>(types[i], List.of(annotations[i]))); } return ret; } diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java index ca10072c6c9..8b5fb128e2b 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentRegistryNode.java @@ -8,10 +8,8 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.ConfigInstance; import com.yahoo.vespa.config.ConfigKey; -import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * @author Tony Vaagenes @@ -20,7 +18,7 @@ import java.util.stream.Collectors; */ public class ComponentRegistryNode extends Node { - private static ComponentId componentRegistryNamespace = ComponentId.fromString("ComponentRegistry"); + private static final ComponentId componentRegistryNamespace = ComponentId.fromString("ComponentRegistry"); private final Class<?> componentClass; @@ -63,7 +61,7 @@ public class ComponentRegistryNode extends Node { @Override public Set<ConfigKey<ConfigInstance>> configKeys() { - return Collections.emptySet(); + return Set.of(); } @Override @@ -76,8 +74,7 @@ public class ComponentRegistryNode extends Node { @Override public boolean equals(Object other) { - if (other instanceof ComponentRegistryNode) { - ComponentRegistryNode that = (ComponentRegistryNode) other; + if (other instanceof ComponentRegistryNode that) { return this.componentId().equals(that.componentId()) && this.instanceType().equals(that.instanceType()) && equalNodeEdges(this.usedComponents(), that.usedComponents()); } else { diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java index 936eadb0b3c..8e6a39a4921 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/GuiceNode.java @@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.vespa.config.ConfigKey; import java.lang.annotation.Annotation; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -24,8 +23,7 @@ public final class GuiceNode extends Node { private final Object myInstance; private final Annotation annotation; - public GuiceNode(Object myInstance, - Annotation annotation) { + public GuiceNode(Object myInstance, Annotation annotation) { super(componentId(myInstance)); this.myInstance = myInstance; this.annotation = annotation; @@ -33,7 +31,7 @@ public final class GuiceNode extends Node { @Override public Set<ConfigKey<ConfigInstance>> configKeys() { - return Collections.emptySet(); + return Set.of(); } @Override @@ -54,7 +52,7 @@ public final class GuiceNode extends Node { @Override public List<Node> usedComponents() { - return Collections.emptyList(); + return List.of(); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java index f8fa0756cb3..6364eab6d42 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java @@ -6,13 +6,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.BLACK; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.GRAY; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.WHITE; import static java.util.logging.Level.FINE; -import static java.util.Collections.singletonList; /** @@ -55,7 +53,7 @@ public class CycleFinder<T> { resetState(); for (T vertex : graph.getVertices()) { if (colors.get(vertex) == WHITE) { - if (visitDepthFirst(vertex, new ArrayList<>(singletonList(vertex)))) { + if (visitDepthFirst(vertex, new ArrayList<>(List.of(vertex)))) { if (cycle == null) throw new IllegalStateException("Null cycle - this should never happen"); if (cycle.isEmpty()) throw new IllegalStateException("Empty cycle - this should never happen"); log.log(FINE, () -> "Cycle detected: " + cycle); diff --git a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java index 872ad82f746..a493905e6d0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java @@ -19,7 +19,7 @@ import java.util.concurrent.Executor; /** * Exposes access log through http. * - * @author dybis + * @author Haakon Dybdahl */ public class AccessLogRequestHandler extends ThreadedHttpRequestHandler { diff --git a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java index 020022dc9fd..10f8400fa37 100644 --- a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java +++ b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java @@ -22,12 +22,11 @@ import com.yahoo.jdisc.http.filter.chain.ResponseFilterChain; import com.yahoo.processing.execution.chain.ChainRegistry; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.logging.Logger; -import static java.util.Collections.emptyList; +import static java.util.List.of; import static java.util.stream.Collectors.toSet; /** @@ -158,7 +157,7 @@ public class FilterChainRepository extends AbstractComponent { } private static Object wrapIfSecurityFilter(Object filter) { - if (isSecurityFilter(filter)) return createSecurityChain(Collections.singletonList(filter)); + if (isSecurityFilter(filter)) return createSecurityChain(List.of(filter)); return filter; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java index 1c325654f2e..71449d6edae 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java @@ -412,7 +412,7 @@ public class HttpRequest { Map<String, String> mask; Map<String, String> view; - mask = Objects.requireNonNullElse(parameterMask, Collections.emptyMap()); + mask = Objects.requireNonNullElse(parameterMask, Map.of()); view = new HashMap<>(parameters.size() + mask.size()); for (Map.Entry<String, List<String>> parameter : parameters.entrySet()) { if (existsAsOriginalParameter(parameter.getValue())) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index 165330662a9..894a231f9be 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -25,7 +25,6 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -91,7 +90,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { @Override protected Iterable<ByteBuffer> responseContent() { - return Collections.singleton(ByteBuffer.wrap(buildMetricOutput(request.getUri().getQuery()))); + return Set.of(ByteBuffer.wrap(buildMetricOutput(request.getUri().getQuery()))); } }.dispatch(handler); @@ -160,7 +159,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } private List<JsonNode> getPacketsForSnapshot(MetricSnapshot metricSnapshot, String application, long timestamp) { - if (metricSnapshot == null) return Collections.emptyList(); + if (metricSnapshot == null) return List.of(); List<JsonNode> packets = new ArrayList<>(); @@ -178,9 +177,9 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } private List<JsonNode> getPacketsForSnapshot(MetricSnapshot metricSnapshot, String metricSetId, String application, long timestamp) { - if (metricSnapshot == null) return Collections.emptyList(); + if (metricSnapshot == null) return List.of(); if (metricSetId == null) return getPacketsForSnapshot(metricSnapshot, application, timestamp); - Set<String> configuredMetrics = metricSets.getOrDefault(metricSetId, Collections.emptySet()); + Set<String> configuredMetrics = metricSets.getOrDefault(metricSetId, Set.of()); List<JsonNode> packets = new ArrayList<>(); for (Map.Entry<MetricDimensions, MetricSet> snapshotEntry : metricSnapshot) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java index ebd5c38e3a3..32fd1d64129 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -31,7 +31,6 @@ import java.io.PrintStream; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -373,11 +372,19 @@ public class StateHandler extends AbstractRequestHandler implements CapabilityRe private boolean isPrometheusRequest(String query) { if (query == null) return false; - return Arrays.asList(query.split("&")).contains("format=prometheus"); + return List.of(query.split("&")).contains("format=prometheus"); } private String prometheusSanitizedName(String name) { - return name.replaceAll("\\.", "_"); + var stringBuilder = new StringBuilder(); + for (char c : name.toCharArray()) { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + stringBuilder.append(c); + } else { + stringBuilder.append("_"); + } + } + return stringBuilder.toString(); } private String sanitizeIfDouble(Number num) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java index 4cb19653848..6d6635ecc94 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricContext.java @@ -3,10 +3,11 @@ package com.yahoo.container.jdisc.state; import com.yahoo.jdisc.Metric; -import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Objects; + +import static java.util.stream.Collectors.toUnmodifiableMap; /** * A context implementation whose identity is the key and values such that this can be used as @@ -41,16 +42,10 @@ public final class StateMetricContext implements MetricDimensions, Metric.Contex } public static StateMetricContext newInstance(Map<String, ?> properties) { - Map<String, String> data; - if (properties != null) { - data = new HashMap<>(properties.size()); - for (Map.Entry<String, ?> entry : properties.entrySet()) { - data.put(entry.getKey(), entry.getValue() != null ? entry.getValue().toString() : null); - } - data = Collections.unmodifiableMap(data); - } else { - data = Collections.emptyMap(); - } + Map<String, String> data = (properties != null) + ? properties.entrySet().stream().collect( + toUnmodifiableMap(Map.Entry::getKey, e -> Objects.requireNonNullElse(e.getValue(), "").toString())) + : Map.of(); return new StateMetricContext(data); } diff --git a/container-core/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java b/container-core/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java index ba143794728..5d96f4d68c1 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java +++ b/container-core/src/main/java/com/yahoo/container/logging/CircularArrayAccessLogKeeper.java @@ -9,7 +9,7 @@ import java.util.List; /** * This class keeps some information from the access log from the requests in memory. It is thread-safe. * - * @author dybis + * @author Haakon Dybdahl */ public class CircularArrayAccessLogKeeper { public static final int SIZE = 1000; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/CookieHelper.java b/container-core/src/main/java/com/yahoo/jdisc/http/CookieHelper.java index e3c2d20ba51..1c9e6572d63 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/CookieHelper.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/CookieHelper.java @@ -3,8 +3,6 @@ package com.yahoo.jdisc.http; import com.yahoo.jdisc.HeaderFields; -import java.util.Arrays; -import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -20,7 +18,7 @@ public class CookieHelper { public static List<Cookie> decodeSetCookieHeader(HeaderFields headers) { List<String> cookies = headers.get(HttpHeaders.Names.SET_COOKIE); if (cookies == null) { - return Collections.emptyList(); + return List.of(); } List<Cookie> ret = new LinkedList<>(); for (String cookie : cookies) { @@ -32,7 +30,7 @@ public class CookieHelper { public static void encodeSetCookieHeader(HeaderFields headers, List<Cookie> cookies) { headers.remove(HttpHeaders.Names.SET_COOKIE); for (Cookie cookie : cookies) { - headers.add(HttpHeaders.Names.SET_COOKIE, Cookie.toSetCookieHeaders(Arrays.asList(cookie))); + headers.add(HttpHeaders.Names.SET_COOKIE, Cookie.toSetCookieHeaders(List.of(cookie))); } } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index 4f1a0bf0d03..6a25937592b 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -16,7 +16,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; import java.security.Principal; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -89,7 +88,11 @@ public class HttpRequest extends Request { this.version = version; this.remoteAddress = remoteAddress; this.parameters.putAll(getUriQueryParameters(uri)); - this.connectedAt = (connectedAtMillis != null) ? connectedAtMillis : creationTime(TimeUnit.MILLISECONDS); + if (connectedAtMillis != null) { + this.connectedAt = connectedAtMillis; + } else { + this.connectedAt = creationTime(TimeUnit.MILLISECONDS); + } } catch (Throwable e) { release(); throw e; @@ -226,7 +229,7 @@ public class HttpRequest extends Request { public List<Cookie> decodeCookieHeader() { List<String> cookies = headers().get(HttpHeaders.Names.COOKIE); if (cookies == null) { - return Collections.emptyList(); + return List.of(); } List<Cookie> ret = new LinkedList<>(); for (String cookie : cookies) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java index a1ea14d80a3..c22e4afec1c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java @@ -242,7 +242,7 @@ public class DiscFilterRequest { public List<String> getHeadersAsList(String name) { List<String> values = parent.headers().get(name); if(values == null) { - return Collections.emptyList(); + return List.of(); } return parent.headers().get(name); } @@ -401,7 +401,7 @@ public class DiscFilterRequest { return Optional.ofNullable(parent.context().get(RequestUtils.JDISC_REQUEST_X509CERT)) .map(X509Certificate[].class::cast) .map(Arrays::asList) - .orElse(Collections.emptyList()); + .orElse(List.of()); } public void setUserRoles(String[] roles) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java index f96bb8b30ae..5070ef75c4d 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java @@ -9,7 +9,6 @@ import com.yahoo.jdisc.http.HttpResponse; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.List; @@ -158,7 +157,7 @@ public class DiscFilterResponse { public void setCookie(String name, String value) { Cookie cookie = new Cookie(name, value); - setCookies(Arrays.asList(cookie)); + setCookies(List.of(cookie)); } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChain.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChain.java index 22874d2f987..dc8eb7daa7a 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChain.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityRequestFilterChain.java @@ -9,7 +9,6 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpRequest; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -42,7 +41,7 @@ public final class SecurityRequestFilterChain extends AbstractResource implement } public static RequestFilter newInstance(SecurityRequestFilter... filters) { - return newInstance(Arrays.asList(filters)); + return newInstance(List.of(filters)); } public static RequestFilter newInstance(List<? extends SecurityRequestFilter> filters) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChain.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChain.java index e6d5e67bc57..16565cb5010 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChain.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/SecurityResponseFilterChain.java @@ -8,8 +8,6 @@ import com.yahoo.jdisc.http.HttpRequest; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -20,12 +18,14 @@ import java.util.Optional; */ public class SecurityResponseFilterChain extends AbstractResource implements ResponseFilter { - private final List<SecurityResponseFilter> filters = new ArrayList<>(); + private final List<SecurityResponseFilter> filters; private SecurityResponseFilterChain(Iterable<? extends SecurityResponseFilter> filters) { + List<SecurityResponseFilter> builder = new ArrayList<>(); for (SecurityResponseFilter filter : filters) { - this.filters.add(filter); + builder.add(filter); } + this.filters = List.copyOf(builder); } @Override @@ -42,7 +42,7 @@ public class SecurityResponseFilterChain extends AbstractResource implements Res } public static ResponseFilter newInstance(SecurityResponseFilter... filters) { - return newInstance(Arrays.asList(filters)); + return newInstance(List.of(filters)); } public static ResponseFilter newInstance(List<? extends SecurityResponseFilter> filters) { @@ -51,7 +51,7 @@ public class SecurityResponseFilterChain extends AbstractResource implements Res /** Returns an unmodifiable view of the filters in this */ public List<SecurityResponseFilter> getFilters() { - return Collections.unmodifiableList(filters); + return filters; } static class RequestViewImpl implements RequestView { @@ -74,7 +74,7 @@ public class SecurityResponseFilterChain extends AbstractResource implements Res @Override public List<String> getHeaders(String name) { List<String> headers = request.headers().get(name); - return headers == null ? Collections.emptyList() : Collections.unmodifiableList(headers); + return headers == null ? List.of() : List.copyOf(headers); } @Override diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/RequestFilterChain.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/RequestFilterChain.java index 2f50b5e319b..d6666060dbc 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/RequestFilterChain.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/RequestFilterChain.java @@ -8,7 +8,6 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.RequestFilter; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -40,7 +39,7 @@ public final class RequestFilterChain extends AbstractResource implements Reques } public static RequestFilter newInstance(RequestFilter... filters) { - return newInstance(Arrays.asList(filters)); + return newInstance(List.of(filters)); } public static RequestFilter newInstance(List<? extends RequestFilter> filters) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/ResponseFilterChain.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/ResponseFilterChain.java index 02d94a7529e..5145c7210f1 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/ResponseFilterChain.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/chain/ResponseFilterChain.java @@ -8,7 +8,6 @@ import com.yahoo.jdisc.application.ResourcePool; import com.yahoo.jdisc.http.filter.ResponseFilter; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -39,7 +38,7 @@ public final class ResponseFilterChain extends AbstractResource implements Respo } public static ResponseFilter newInstance(ResponseFilter... filters) { - return newInstance(Arrays.asList(filters)); + return newInstance(List.of(filters)); } public static ResponseFilter newInstance(List<? extends ResponseFilter> filters) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/CompletionHandlers.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/CompletionHandlers.java index caa942b6a5c..23efec5f98d 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/CompletionHandlers.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/CompletionHandlers.java @@ -3,7 +3,8 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.handler.CompletionHandler; -import java.util.Arrays; +import java.util.List; + /** * @author Simon Thoresen Hult @@ -33,7 +34,7 @@ public class CompletionHandlers { } public static CompletionHandler wrap(CompletionHandler... handlers) { - return wrap(Arrays.asList(handlers)); + return wrap(List.of(handlers)); } public static CompletionHandler wrap(final Iterable<CompletionHandler> handlers) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java index b43281c524a..7e6b9b822ac 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/FormPostRequestHandler.java @@ -20,7 +20,6 @@ import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; import java.nio.charset.UnsupportedCharsetException; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -134,7 +133,7 @@ class FormPostRequestHandler extends AbstractRequestHandler implements ContentCh */ private static Map<String, List<String>> parseFormParameters(String formContent) { if (formContent.isEmpty()) { - return Collections.emptyMap(); + return Map.of(); } Map<String, List<String>> parameterMap = new HashMap<>(); diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/Point.java b/container-core/src/main/java/com/yahoo/metrics/simple/Point.java index 5c7abe03596..4a1ae80fc69 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/Point.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/Point.java @@ -74,13 +74,11 @@ public final class Point implements Context { @Override public String toString() { final int maxLen = 3; - StringBuilder builder = new StringBuilder(); - builder.append("Point [location=") - .append(Arrays.asList(location).subList(0, Math.min(location.length, maxLen))) - .append(", dimensions=") - .append(Arrays.asList(dimensions).subList(0, Math.min(dimensions.length, maxLen))) - .append("]"); - return builder.toString(); + return "Point [location=" + + List.of(location).subList(0, Math.min(location.length, maxLen)) + + ", dimensions=" + + List.of(dimensions).subList(0, Math.min(dimensions.length, maxLen)) + + "]"; } /** diff --git a/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java index e8a92211afb..decf7a557be 100644 --- a/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java +++ b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java @@ -7,7 +7,6 @@ import com.yahoo.jdisc.test.NonWorkingOsgiFramework; import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; -import java.util.Collections; import java.util.List; /** @@ -23,7 +22,7 @@ public class MockOsgi extends NonWorkingOsgiFramework implements Osgi { @Override public List<Bundle> getCurrentBundles() { - return Collections.emptyList(); + return List.of(); } @Override @@ -33,7 +32,7 @@ public class MockOsgi extends NonWorkingOsgiFramework implements Osgi { @Override public List<Bundle> install(String absolutePath) { - return Collections.emptyList(); + return List.of(); } } diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java index cd22dc80a4c..b17e4229c4c 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java @@ -12,9 +12,9 @@ import com.yahoo.processing.Processor; import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.processing.rendering.Renderer; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -38,12 +38,12 @@ public class ProcessingTestDriver extends RequestHandlerTestDriver { @SafeVarargs @SuppressWarnings("varargs") public ProcessingTestDriver(Chain<Processor> ... chains) { - this(Arrays.asList(chains), new ComponentRegistry<>()); + this(List.of(chains), new ComponentRegistry<>()); } @SafeVarargs @SuppressWarnings("varargs") public ProcessingTestDriver(String binding, Chain<Processor> ... chains) { - this(binding, Arrays.asList(chains), new ComponentRegistry<>()); + this(binding, List.of(chains), new ComponentRegistry<>()); } public ProcessingTestDriver(Collection<Chain<Processor>> chains, ComponentRegistry<Renderer> renderers) { this(createProcessingHandler(chains, renderers, AccessLog.voidAccessLog())); diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index cb4743cd83e..b4536a1c56b 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -173,7 +173,7 @@ public final class CompoundName { if (isEmpty()) return fromComponents(nameParts); List<String> newCompounds = new ArrayList<>(nameParts.length + compounds.size()); - newCompounds.addAll(Arrays.asList(nameParts)); + newCompounds.addAll(List.of(nameParts)); newCompounds.addAll(this.compounds); return new CompoundName(newCompounds); } @@ -204,7 +204,7 @@ public final class CompoundName { throw new IllegalArgumentException("Asked for the first " + n + " components but '" + this + "' only have " + compounds.size() + " components."); if (compounds.size() == n) return this; - if (compounds.size() == 0) return empty; + if (compounds.isEmpty()) return empty; if (compounds.size() - 1 == n) return first; return first.first(n); } @@ -316,7 +316,7 @@ public final class CompoundName { for (String compound : compounds) all += compound.length(); StringBuilder b = new StringBuilder(all); for (String compound : compounds) b.append(compound).append("."); - return b.length()==0 ? "" : b.substring(0, b.length()-1); + return b.isEmpty() ? "" : b.substring(0, b.length()-1); } /** diff --git a/container-core/src/main/java/com/yahoo/processing/request/Properties.java b/container-core/src/main/java/com/yahoo/processing/request/Properties.java index d06e8b26cd9..e689ea76f3a 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/Properties.java +++ b/container-core/src/main/java/com/yahoo/processing/request/Properties.java @@ -3,7 +3,6 @@ package com.yahoo.processing.request; import java.util.Map; import java.util.HashMap; -import java.util.Collections; /** * The properties of a request @@ -271,7 +270,7 @@ public class Properties implements Cloneable { * @throws RuntimeException if no instance in the chain accepted this name-value pair */ public final void clearAll(String name) { - clearAll(new CompoundName(name), Collections.<String,String>emptyMap()); + clearAll(new CompoundName(name), Map.of()); } /** diff --git a/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java b/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java index d820c2812e5..ee2e920afcc 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java +++ b/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java @@ -5,7 +5,6 @@ import com.yahoo.collections.Tuple2; import com.yahoo.concurrent.CompletableFutures; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -60,13 +59,13 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< /** Adds new data and marks this as completed */ @Override public synchronized void addLast(DATATYPE data) { - addLast(Collections.singletonList(data)); + addLast(List.of(data)); } /** Adds new data without completing this */ @Override public synchronized void add(DATATYPE data) { - add(Collections.singletonList(data)); + add(List.of(data)); } /** Adds new data and marks this as completed */ diff --git a/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java b/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java index 45dec0ad2ba..fb891f0b8ea 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java +++ b/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java @@ -3,7 +3,6 @@ package com.yahoo.processing.response; import com.yahoo.processing.impl.ProcessingFuture; -import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -158,7 +157,7 @@ public interface IncomingData<DATATYPE extends Data> { } public List<DATATYPE> drain() { - return Collections.emptyList(); + return List.of(); } /** diff --git a/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java b/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java index 8c94b5f9a93..207e18084ab 100644 --- a/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java +++ b/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilderTest.java @@ -11,8 +11,6 @@ import com.yahoo.component.chain.dependencies.Provides; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -90,15 +88,15 @@ public class ChainBuilderTest { @Test void testPhaseAndSearcher() { ChainBuilder depHandler = newChainBuilder(); - depHandler.addPhase(new Phase("phase1", set("phase2"), Collections.<String>emptySet())); + depHandler.addPhase(new Phase("phase1", set("phase2"), Set.of())); depHandler.addPhase(new Phase("phase2", set("phase3"), set("phase1"))); - depHandler.addPhase(new Phase("phase3", Collections.<String>emptySet(), set("phase2", "phase1"))); + depHandler.addPhase(new Phase("phase3", Set.of(), set("phase2", "phase1"))); ChainedComponent first = new First(); ChainedComponent second = new Second(); depHandler.addComponent(first); depHandler.addComponent(second); - assertEquals(depHandler.orderNodes().components(), Arrays.asList(first, second)); + assertEquals(depHandler.orderNodes().components(), List.of(first, second)); } @@ -114,7 +112,7 @@ public class ChainBuilderTest { chainBuilder.addComponent(c); chainBuilder.addComponent(a2); - assertEquals(Arrays.asList(a1, c, a2), chainBuilder.orderNodes().components()); + assertEquals(List.of(a1, c, a2), chainBuilder.orderNodes().components()); } private ChainBuilder newChainBuilder() { @@ -122,7 +120,7 @@ public class ChainBuilderTest { } private Set<String> set(String... strings) { - return new HashSet<>(Arrays.asList(strings)); + return new HashSet<>(List.of(strings)); } @Before("phase1") @@ -172,9 +170,9 @@ public class ChainBuilderTest { private ChainBuilder createDependencyHandler() { ChainBuilder chainBuilder = newChainBuilder(); - chainBuilder.addPhase(new Phase("phase1", Collections.<String>emptySet(), Collections.<String>emptySet())); - chainBuilder.addPhase(new Phase("phase2", Collections.<String>emptySet(), Collections.<String>emptySet())); - chainBuilder.addPhase(new Phase("phase3", Collections.<String>emptySet(), Collections.<String>emptySet())); + chainBuilder.addPhase(new Phase("phase1", Set.of(), Set.of())); + chainBuilder.addPhase(new Phase("phase2", Set.of(), Set.of())); + chainBuilder.addPhase(new Phase("phase3", Set.of(), Set.of())); return chainBuilder; } diff --git a/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java b/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java index f1dd1ff03c0..126e96c7543 100644 --- a/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java +++ b/container-core/src/test/java/com/yahoo/component/chain/dependencies/ordering/OrderedReadyNodesTest.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.component.chain.dependencies.ordering; -import java.util.Arrays; import com.yahoo.component.chain.ChainedComponent; @@ -11,6 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import com.yahoo.component.ComponentId; +import java.util.List; + /** * Test for OrderedReadyNodes. @@ -20,14 +21,14 @@ import com.yahoo.component.ComponentId; @SuppressWarnings("rawtypes") public class OrderedReadyNodesTest { - class ComponentA extends ChainedComponent { + static class ComponentA extends ChainedComponent { public ComponentA(ComponentId id) { super(id); } @Override public Dependencies getDependencies() { - return new Dependencies(Arrays.asList(getId().getName()), null, null); + return new Dependencies(List.of(getId().getName()), null, null); } } diff --git a/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java b/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java index 14349d4cd51..7240b6510c5 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/ConfigRetrieverTest.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; -import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -45,7 +44,7 @@ public class ConfigRetrieverTest { void require_that_bootstrap_configs_come_first() { writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); - ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Collections.emptySet(), 0, true); + ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Set.of(), 0, true); assertTrue(bootstrapConfigs instanceof BootstrapConfigs); retriever.shutdown(); @@ -56,10 +55,10 @@ public class ConfigRetrieverTest { void require_that_components_comes_after_bootstrap() { writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); - ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Collections.emptySet(), 0, true); + ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Set.of(), 0, true); ConfigKey<? extends ConfigInstance> testConfigKey = new ConfigKey<>(TestConfig.class, dirConfigSource.configId()); - ConfigSnapshot componentsConfigs = retriever.getConfigs(Collections.singleton(testConfigKey), 0, true); + ConfigSnapshot componentsConfigs = retriever.getConfigs(Set.of(testConfigKey), 0, true); if (componentsConfigs instanceof ComponentsConfigs) { assertEquals(3, componentsConfigs.size()); @@ -75,8 +74,8 @@ public class ConfigRetrieverTest { writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); ConfigKey<? extends ConfigInstance> testConfigKey = new ConfigKey<>(TestConfig.class, dirConfigSource.configId()); - ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Collections.emptySet(), 0, true); - ConfigSnapshot componentsConfigs = retriever.getConfigs(Collections.singleton(testConfigKey), 0, true); + ConfigSnapshot bootstrapConfigs = retriever.getConfigs(Set.of(), 0, true); + ConfigSnapshot componentsConfigs = retriever.getConfigs(Set.of(testConfigKey), 0, true); Set<ConfigKey<? extends ConfigInstance>> keys = new HashSet<>(); keys.add(testConfigKey); keys.add(new ConfigKey<>(TestConfig.class, "")); @@ -92,8 +91,8 @@ public class ConfigRetrieverTest { void require_that_empty_components_keys_after_bootstrap_returns_components_configs() { writeConfigs(); ConfigRetriever retriever = createConfigRetriever(); - assertTrue(retriever.getConfigs(Collections.emptySet(), 0, true) instanceof BootstrapConfigs); - assertTrue(retriever.getConfigs(Collections.emptySet(), 0, true) instanceof ComponentsConfigs); + assertTrue(retriever.getConfigs(Set.of(), 0, true) instanceof BootstrapConfigs); + assertTrue(retriever.getConfigs(Set.of(), 0, true) instanceof ComponentsConfigs); retriever.shutdown(); } diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index aacc43d725b..e99f98cb842 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -21,9 +21,9 @@ import com.yahoo.vespa.config.ConfigKey; import org.junit.jupiter.api.Test; import java.lang.annotation.Annotation; -import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -480,7 +480,7 @@ public class ComponentGraphTest { ComponentGraph graph = new ComponentGraph(); graph.add(mockComponentNodeWithId(ExecutorProvider.class, "dummyId")); graph.complete(); - graph.setAvailableConfigs(Collections.emptyMap()); + graph.setAvailableConfigs(Map.of()); return graph; } @@ -607,7 +607,7 @@ public class ComponentGraphTest { } public static class ExecutorProvider implements Provider<Executor> { - private Executor executor = Executors.newSingleThreadExecutor(); + private final Executor executor = Executors.newSingleThreadExecutor(); public Executor get() { return executor; diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java index 59c939863dd..17ead389c48 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ReuseComponentsTest.java @@ -16,7 +16,7 @@ import com.yahoo.container.di.componentgraph.core.ComponentGraphTest.SimpleCompo import com.yahoo.vespa.config.ConfigKey; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.Map; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.Supplier; @@ -68,12 +68,12 @@ public class ReuseComponentsTest { Class<ComponentTakingConfig> componentClass = ComponentTakingConfig.class; ComponentGraph graph = buildGraph(componentClass); - graph.setAvailableConfigs(Collections.singletonMap(new ConfigKey<>(TestConfig.class, "component"), + graph.setAvailableConfigs(Map.of(new ConfigKey<>(TestConfig.class, "component"), ConfigGetter.getConfig(TestConfig.class, "raw: stringVal \"oldConfig\""))); ComponentTakingConfig instance = getComponent(graph, componentClass); ComponentGraph newGraph = buildGraph(componentClass); - newGraph.setAvailableConfigs(Collections.singletonMap(new ConfigKey<>(TestConfig.class, "component"), + newGraph.setAvailableConfigs(Map.of(new ConfigKey<>(TestConfig.class, "component"), ConfigGetter.getConfig(TestConfig.class, "raw: stringVal \"newConfig\""))); newGraph.reuseNodes(graph); ComponentTakingConfig instance2 = getComponent(newGraph, componentClass); @@ -98,7 +98,7 @@ public class ReuseComponentsTest { graph.add(injectedComponent); graph.complete(); - graph.setAvailableConfigs(Collections.singletonMap(new ConfigKey<>(TestConfig.class, configId), + graph.setAvailableConfigs(Map.of(new ConfigKey<>(TestConfig.class, configId), ConfigGetter.getConfig(TestConfig.class, "raw: stringVal \"" + config + "\""))); return graph; @@ -132,7 +132,7 @@ public class ReuseComponentsTest { } graph.complete(); - graph.setAvailableConfigs(Collections.emptyMap()); + graph.setAvailableConfigs(Map.of()); return graph; }; @@ -163,7 +163,7 @@ public class ReuseComponentsTest { graph.add(injectedComponent); graph.complete(); - graph.setAvailableConfigs(Collections.singletonMap(new ConfigKey<>(TestConfig.class, configId), + graph.setAvailableConfigs(Map.of(new ConfigKey<>(TestConfig.class, configId), ConfigGetter.getConfig(TestConfig.class, "raw: stringVal \"" + config + "\""))); return graph; @@ -188,7 +188,7 @@ public class ReuseComponentsTest { ComponentGraph graph = new ComponentGraph(); graph.add(mockComponentNode(ComponentTakingExecutor.class, "dummyId")); graph.complete(ComponentGraphTest.singletonExecutorInjector); - graph.setAvailableConfigs(Collections.emptyMap()); + graph.setAvailableConfigs(Map.of()); return graph; }; @@ -224,7 +224,7 @@ public class ReuseComponentsTest { private void completeNode(ComponentNode node) { node.setArguments(new Object[0]); - node.setAvailableConfigs(Collections.emptyMap()); + node.setAvailableConfigs(Map.of()); } private ComponentGraph buildGraph(Class<?> componentClass) { @@ -237,7 +237,7 @@ public class ReuseComponentsTest { private ComponentGraph buildGraphAndSetNoConfigs(Class<?> componentClass) { ComponentGraph g = buildGraph(componentClass); - g.setAvailableConfigs(Collections.emptyMap()); + g.setAvailableConfigs(Map.of()); return g; } diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java index 3d5375a9740..11de808a415 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java @@ -116,7 +116,7 @@ public class ContainerThreadPoolImplTest { assertEquals(CPUS * 4 * 100, executor.getQueue().remainingCapacity()); } - private class FlipIt implements Runnable { + private static class FlipIt implements Runnable { public final Receiver<Boolean> didItRun = new Receiver<>(); @Override @@ -176,7 +176,7 @@ public class ContainerThreadPoolImplTest { } } - private class Hang implements Runnable { + private static class Hang implements Runnable { private final long hangMillis; diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/HttpRequestTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/HttpRequestTestCase.java index f70772b7331..2aa5c888865 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/HttpRequestTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/HttpRequestTestCase.java @@ -8,7 +8,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; +import java.util.Map; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -30,7 +30,7 @@ public class HttpRequestTestCase { @BeforeEach public void setUp() throws Exception { requestData = new ByteArrayInputStream(Utf8.toBytes(X_RAY_YANKEE_ZULU)); - r = HttpRequest.createTestRequest(HTTP_MAILHOST_25_ALPHA_BRAVO_CHARLIE_DELTA, Method.GET, requestData, Collections.singletonMap("foxtrot", "golf")); + r = HttpRequest.createTestRequest(HTTP_MAILHOST_25_ALPHA_BRAVO_CHARLIE_DELTA, Method.GET, requestData, Map.of("foxtrot", "golf")); } @AfterEach diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricSnapshotTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricSnapshotTest.java index 3217bff4bb3..ccc7b22f348 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricSnapshotTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricSnapshotTest.java @@ -3,8 +3,8 @@ package com.yahoo.container.jdisc.state; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -29,8 +29,8 @@ public class MetricSnapshotTest { @Test void testEquality() { - assertEquals(Collections.unmodifiableMap(new HashMap(0)).hashCode(), Collections.emptyMap().hashCode()); - assertEquals(Collections.unmodifiableMap(new HashMap(0)), Collections.emptyMap()); + assertEquals(new HashMap(0).hashCode(), Map.of().hashCode()); + assertEquals(new HashMap(0), Map.of()); } } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java index c99a61781cb..0aa2b0f41d5 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java @@ -79,10 +79,8 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void testPrometheusFormat() { - var counterContext = StateMetricContext.newInstance(Map.of("label1", "val1", "label2", "val2")); - var otherContext = StateMetricContext.newInstance(Map.of( - "label1", "This label has \"quotes\"", - "label2", "This label, a\nnewline")); + var counterContext = StateMetricContext.newInstance(Map.of("label", "This label has \"quotes\"")); + var otherContext = StateMetricContext.newInstance(Map.of("label", "This label, a\nnewline")); var snapshot = new MetricSnapshot(0L, SNAPSHOT_INTERVAL, TimeUnit.MILLISECONDS); snapshot.add(counterContext, "some.counter", 10); snapshot.add(counterContext, "some.counter", 20); @@ -90,7 +88,7 @@ public class StateHandlerTest extends StateHandlerTestBase { snapshot.add(otherContext, "some.counter", 2); snapshot.set(null, "bar", 20); snapshot.set(null, "bar", 40); - snapshot.set(null, "testing.infinity", Double.NEGATIVE_INFINITY); + snapshot.set(null, "testing-infinity", Double.NEGATIVE_INFINITY); snapshot.set(null, "testing.nan", Double.NaN); snapshotProvider.setSnapshot(snapshot); @@ -100,8 +98,8 @@ public class StateHandlerTest extends StateHandlerTestBase { bar_count 2 300000 bar_max 40.0 300000 bar_sum 60.0 300000 - some_counter_count{label1="This label has \\"quotes\\"",label2="This label, a\\nnewline",} 3 300000 - some_counter_count{label1="val1",label2="val2",} 30 300000 + some_counter_count{label="This label has \\"quotes\\"",} 30 300000 + some_counter_count{label="This label, a\\nnewline",} 3 300000 testing_infinity_count 1 300000 testing_infinity_max -Inf 300000 testing_infinity_sum -Inf 300000 diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java b/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java index a629acd0d1b..6f5af2f308b 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java @@ -8,8 +8,6 @@ import org.junit.jupiter.api.Test; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -97,11 +95,11 @@ public class HttpRequestTestCase { request.setConnectionTimeout(1, TimeUnit.SECONDS); assertEquals(Long.valueOf(1000), request.getConnectionTimeout(TimeUnit.MILLISECONDS)); - assertEquals(Arrays.asList("bar", "baz"), request.parameters().get("foo")); - assertEquals(Collections.singletonList("69"), request.parameters().get("cox")); - request.parameters().put("cox", Arrays.asList("6", "9")); - assertEquals(Arrays.asList("bar", "baz"), request.parameters().get("foo")); - assertEquals(Arrays.asList("6", "9"), request.parameters().get("cox")); + assertEquals(List.of("bar", "baz"), request.parameters().get("foo")); + assertEquals(List.of("69"), request.parameters().get("cox")); + request.parameters().put("cox", List.of("6", "9")); + assertEquals(List.of("bar", "baz"), request.parameters().get("foo")); + assertEquals(List.of("6", "9"), request.parameters().get("cox")); assertEquals(1L, request.getConnectedAt(TimeUnit.MILLISECONDS)); } @@ -170,7 +168,7 @@ public class HttpRequestTestCase { @Test void requireThatCookieHeaderCanBeEncoded() throws Exception { final HttpRequest request = newRequest(HttpRequest.Version.HTTP_1_0); - final List<Cookie> cookies = Collections.singletonList(new Cookie("foo", "bar")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar")); request.encodeCookieHeader(cookies); final List<String> headers = request.headers().get(com.yahoo.jdisc.http.HttpHeaders.Names.COOKIE); assertEquals(1, headers.size()); @@ -180,7 +178,7 @@ public class HttpRequestTestCase { @Test void requireThatCookieHeaderCanBeDecoded() throws Exception { final HttpRequest request = newRequest(HttpRequest.Version.HTTP_1_0); - final List<Cookie> cookies = Collections.singletonList(new Cookie("foo", "bar")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar")); request.encodeCookieHeader(cookies); assertEquals(cookies, request.decodeCookieHeader()); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java b/container-core/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java index da897794af2..c521462c896 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java @@ -4,8 +4,6 @@ package com.yahoo.jdisc.http; import com.yahoo.jdisc.Response; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -79,7 +77,7 @@ public class HttpResponseTestCase { @Test void requireThatCookieHeaderCanBeEncoded() throws Exception { final HttpResponse response = newResponse(69); - final List<Cookie> cookies = Collections.singletonList(new Cookie("foo", "bar")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar")); response.encodeSetCookieHeader(cookies); final List<String> headers = response.headers().get(HttpHeaders.Names.SET_COOKIE); assertEquals(1, headers.size()); @@ -89,18 +87,18 @@ public class HttpResponseTestCase { @Test void requireThatMultipleCookieHeadersCanBeEncoded() throws Exception { final HttpResponse response = newResponse(69); - final List<Cookie> cookies = Arrays.asList(new Cookie("foo", "bar"), new Cookie("baz", "cox")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar"), new Cookie("baz", "cox")); response.encodeSetCookieHeader(cookies); final List<String> headers = response.headers().get(HttpHeaders.Names.SET_COOKIE); assertEquals(2, headers.size()); - assertEquals(Cookie.toSetCookieHeaders(Arrays.asList(new Cookie("foo", "bar"), new Cookie("baz", "cox"))), + assertEquals(Cookie.toSetCookieHeaders(List.of(new Cookie("foo", "bar"), new Cookie("baz", "cox"))), headers); } @Test void requireThatCookieHeaderCanBeDecoded() throws Exception { final HttpResponse response = newResponse(69); - final List<Cookie> cookies = Collections.singletonList(new Cookie("foo", "bar")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar")); response.encodeSetCookieHeader(cookies); assertEquals(cookies, response.decodeSetCookieHeader()); } @@ -108,7 +106,7 @@ public class HttpResponseTestCase { @Test void requireThatMultipleCookieHeadersCanBeDecoded() throws Exception { final HttpResponse response = newResponse(69); - final List<Cookie> cookies = Arrays.asList(new Cookie("foo", "bar"), new Cookie("baz", "cox")); + final List<Cookie> cookies = List.of(new Cookie("foo", "bar"), new Cookie("baz", "cox")); response.encodeSetCookieHeader(cookies); assertEquals(cookies, response.decodeSetCookieHeader()); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java index 6460dee8365..9a3005c0971 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.List; @@ -306,7 +305,7 @@ public class DiscFilterRequestTest { URI uri = URI.create("http://example.yahoo.com/test"); HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); httpReq.headers().add("key1", "value1"); - httpReq.headers().add("key2", Arrays.asList("value1", "value2")); + httpReq.headers().add("key2", List.of("value1", "value2")); DiscFilterRequest request = new DiscFilterRequest(httpReq); HeaderFields headers = request.getUntreatedHeaders(); diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 9c17ce7d0da..73dfb85519c 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -54,7 +54,6 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -667,7 +666,7 @@ public class HttpServerTest { private ResponseMetricAggregator.StatisticsEntry waitForStatistics(ResponseMetricAggregator statisticsCollector) { - List<ResponseMetricAggregator.StatisticsEntry> entries = Collections.emptyList(); + List<ResponseMetricAggregator.StatisticsEntry> entries = List.of(); int tries = 0; // Wait up to 30 seconds before giving up while (entries.isEmpty() && tries < 300) { @@ -912,7 +911,7 @@ public class HttpServerTest { @Override public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { final HttpResponse response = HttpResponse.newInstance(OK); - response.encodeSetCookieHeader(Collections.singletonList(cookie)); + response.encodeSetCookieHeader(List.of(cookie)); ResponseDispatch.newInstance(response).dispatch(handler); return null; } @@ -922,8 +921,8 @@ public class HttpServerTest { @Override public ContentChannel handleRequest(final Request request, final ResponseHandler handler) { - final List<Cookie> cookies = new ArrayList<>(((HttpRequest)request).decodeCookieHeader()); - Collections.sort(cookies, new CookieComparator()); + List<Cookie> cookies = new ArrayList<>(((HttpRequest)request).decodeCookieHeader()); + cookies.sort(new CookieComparator()); final ContentChannel out = ResponseDispatch.newInstance(Response.Status.OK).connect(handler); out.write(StandardCharsets.UTF_8.encode(cookies.toString()), null); out.close(null); diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/PointTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/PointTest.java index 0368c925bcc..ad95c87ca8c 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/PointTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/PointTest.java @@ -3,8 +3,8 @@ package com.yahoo.metrics.simple; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -15,7 +15,7 @@ public class PointTest { @Test void testPointEquality() { - Point a = new Point(Collections.emptyMap()); + Point a = new Point(Map.of()); Point b = new Point(new HashMap<>(0)); assertEquals(a.hashCode(), b.hashCode()); assertEquals(a, b); diff --git a/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java b/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java index d8743d063fb..40ec5973510 100644 --- a/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java +++ b/container-core/src/test/java/com/yahoo/metrics/simple/jdisc/SnapshotConverterTest.java @@ -14,7 +14,6 @@ import com.yahoo.metrics.simple.Point; import com.yahoo.metrics.simple.UntypedMetric; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -29,7 +28,7 @@ public class SnapshotConverterTest { @Test void testPointConversion() { - MetricDimensions a = SnapshotConverter.convert(new Point(Collections.emptyMap())); + MetricDimensions a = SnapshotConverter.convert(new Point(Map.of())); MetricDimensions b = SnapshotConverter.convert(new Point(new HashMap<>(0))); MetricDimensions c = SnapshotConverter.convert((Point) null); assertEquals(a.hashCode(), b.hashCode()); diff --git a/container-core/src/test/java/com/yahoo/processing/processors/MockUserDatabaseClientTest.java b/container-core/src/test/java/com/yahoo/processing/processors/MockUserDatabaseClientTest.java index 76de0767ef0..0149842393f 100644 --- a/container-core/src/test/java/com/yahoo/processing/processors/MockUserDatabaseClientTest.java +++ b/container-core/src/test/java/com/yahoo/processing/processors/MockUserDatabaseClientTest.java @@ -17,7 +17,7 @@ import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; import java.net.URI; import java.util.Collection; -import java.util.Collections; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -30,7 +30,7 @@ public class MockUserDatabaseClientTest { Request request = null; try { Chain<Processor> chain = new Chain<>("default", new MockUserDatabaseClient()); - setupJDisc(Collections.singletonList(chain)); + setupJDisc(List.of(chain)); request = createRequest(); Response response = Execution.createRoot(chain, 0, Execution.Environment.createEmpty()).process(request); MockUserDatabaseClient.User user = (MockUserDatabaseClient.User) response.data().request().properties().get("User"); diff --git a/container-core/src/test/java/com/yahoo/processing/request/test/PropertyMapTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/test/PropertyMapTestCase.java index 1f1f1d65677..43c7ccc75b2 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/test/PropertyMapTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/test/PropertyMapTestCase.java @@ -5,7 +5,6 @@ import com.yahoo.lang.PublicCloneable; import com.yahoo.processing.request.properties.PropertyMap; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -24,8 +23,8 @@ public class PropertyMapTestCase { map.set("clonableArray", new ClonableObject[]{new ClonableObject()}); map.set("publicClonableArray", new ClonableObject[]{new ClonableObject()}); map.set("nonclonableArray", new NonClonableObject[]{new NonClonableObject()}); - map.set("clonableList", Collections.singletonList(new ClonableObject())); - map.set("nonclonableList", Collections.singletonList(new NonClonableObject())); + map.set("clonableList", List.of(new ClonableObject())); + map.set("nonclonableList", List.of(new NonClonableObject())); assertNotNull(map.get("clonable")); assertNotNull(map.get("nonclonable")); diff --git a/container-core/src/test/java/com/yahoo/processing/test/documentation/Federator.java b/container-core/src/test/java/com/yahoo/processing/test/documentation/Federator.java index 101f5d31648..c7892ddbc1e 100644 --- a/container-core/src/test/java/com/yahoo/processing/test/documentation/Federator.java +++ b/container-core/src/test/java/com/yahoo/processing/test/documentation/Federator.java @@ -10,7 +10,6 @@ import com.yahoo.processing.execution.Execution; import com.yahoo.processing.response.FutureResponse; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -23,7 +22,7 @@ public class Federator extends Processor { @SafeVarargs @SuppressWarnings("varargs") public Federator(Chain<? extends Processor> ... chains) { - this.chains = Arrays.asList(chains); + this.chains = List.of(chains); } @SuppressWarnings("unchecked") diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index a06365abbeb..f67f33a3b05 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -154,6 +154,7 @@ <include>com.microsoft.onnxruntime:onnxruntime:${onnxruntime.vespa.version}:test</include> <include>com.thaiopensource:jing:20091111:test</include> <include>commons-codec:commons-codec:${commons-codec.vespa.version}:test</include> + <include>de.kherud:llama:${kherud.llama.vespa.version}:test</include> <include>io.airlift:aircompressor:${aircompressor.vespa.version}:test</include> <include>io.airlift:airline:${airline.vespa.version}:test</include> <include>io.prometheus:simpleclient:${prometheus.client.vespa.version}:test</include> diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java index 6e8b8d44d0d..f0173fe4d68 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java @@ -13,7 +13,6 @@ import java.time.Instant; import java.util.List; import java.util.function.Supplier; -import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -33,7 +32,7 @@ public class DeconstructorTest { deconstructor = new Deconstructor(); var slowDeconstructComponent = new SlowDeconstructComponent(); - deconstructor.deconstruct(0, List.of(slowDeconstructComponent), emptyList()); + deconstructor.deconstruct(0, List.of(slowDeconstructComponent), List.of()); deconstructor.shutdown(); assertTrue(slowDeconstructComponent.destructed); } @@ -41,7 +40,7 @@ public class DeconstructorTest { @Test void require_abstract_component_destructed() throws InterruptedException { TestAbstractComponent abstractComponent = new TestAbstractComponent(); - deconstructor.deconstruct(0, List.of(abstractComponent), emptyList()); + deconstructor.deconstruct(0, List.of(abstractComponent), List.of()); waitForDeconstructToComplete(() -> abstractComponent.destructed); assertTrue(abstractComponent.destructed); @@ -50,7 +49,7 @@ public class DeconstructorTest { @Test void require_provider_destructed() throws InterruptedException { TestProvider provider = new TestProvider(); - deconstructor.deconstruct(0, List.of(provider), emptyList()); + deconstructor.deconstruct(0, List.of(provider), List.of()); waitForDeconstructToComplete(() -> provider.destructed); assertTrue(provider.destructed); @@ -59,7 +58,7 @@ public class DeconstructorTest { @Test void require_shared_resource_released() throws InterruptedException { TestSharedResource sharedResource = new TestSharedResource(); - deconstructor.deconstruct(0, List.of(sharedResource), emptyList()); + deconstructor.deconstruct(0, List.of(sharedResource), List.of()); waitForDeconstructToComplete(() -> sharedResource.released); assertTrue(sharedResource.released); } @@ -68,7 +67,7 @@ public class DeconstructorTest { void bundles_are_uninstalled() throws InterruptedException { var bundle = new UninstallableMockBundle(); // Done by executor, so it takes some time even with a 0 delay. - deconstructor.deconstruct(0, emptyList(), singleton(bundle)); + deconstructor.deconstruct(0, List.of(), singleton(bundle)); waitForDeconstructToComplete(() -> bundle.uninstalled); assertTrue(bundle.uninstalled); diff --git a/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java b/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java index 11ba05e363d..846a2008858 100644 --- a/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java +++ b/container-llama/src/main/java/ai/vespa/llama/LlamaBundleActivator.java @@ -13,12 +13,19 @@ import java.util.logging.Logger; **/ public class LlamaBundleActivator implements BundleActivator { + private static final String SKIP_SUFFIX = ".skip"; + private static final String SKIP_VALUE = "true"; private static final String PATH_PROPNAME = "de.kherud.llama.lib.path"; private static final Logger log = Logger.getLogger(LlamaBundleActivator.class.getName()); @Override public void start(BundleContext ctx) { log.fine("start bundle"); + String skipAll = LlamaBundleActivator.class.getSimpleName() + SKIP_SUFFIX; + if (SKIP_VALUE.equals(System.getProperty(skipAll))) { + log.info("skip loading of native libraries"); + return; + } if (checkFilenames( "/dev/nvidia0", "/opt/vespa-deps/lib64/cuda/libllama.so", diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java b/container-llama/src/main/java/de/kherud/llama/package-info.java index ac6c913381c..3c9773762b4 100644 --- a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java +++ b/container-llama/src/main/java/de/kherud/llama/package-info.java @@ -1,5 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author lesters + */ @ExportPackage -package com.yahoo.errorhandling; +package de.kherud.llama; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java index 5528085c5b8..bb3f4e0f5b0 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java @@ -14,7 +14,6 @@ import com.yahoo.messagebus.shared.ServerSession; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.test.SimpleProtocol; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -83,7 +82,7 @@ public class ServerTestDriver { if (reply == null) { return null; } - List<Integer> lst = new LinkedList<>(Arrays.asList(errCodes)); + List<Integer> lst = new LinkedList<>(List.of(errCodes)); for (int i = 0, len = reply.getNumErrors(); i < len; ++i) { Error err = reply.getError(i); System.out.println(err); diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index e514453bc08..1c85ec37e34 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -5,11 +5,9 @@ import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.messagebus.network.NetworkMultiplexer; import com.yahoo.messagebus.shared.NullNetwork; -import com.yahoo.vespa.config.content.DistributionConfig; import org.junit.Test; import static org.junit.Assert.assertNotNull; diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index 6cc22beb921..9d1668ba671 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java @@ -24,7 +24,7 @@ import org.junit.Test; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; +import java.util.List; import java.util.concurrent.BlockingDeque; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; @@ -643,7 +643,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override public Iterable<ByteBuffer> newResponseContent() { - return Collections.emptyList(); + return List.of(); } @Override diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java index 58b9ebb108e..cd60d4a30ea 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java @@ -28,7 +28,6 @@ import org.junit.Test; import java.net.URI; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -245,13 +244,13 @@ public class MbusServerTestCase { public void requireThatResponseErrorCodeDoesNotDuplicateReplyError() { assertError(Collections.<Integer>emptyList(), Response.Status.OK); - assertError(Arrays.asList(ErrorCode.APP_FATAL_ERROR), + assertError(List.of(ErrorCode.APP_FATAL_ERROR), Response.Status.BAD_REQUEST); - assertError(Arrays.asList(ErrorCode.FATAL_ERROR), + assertError(List.of(ErrorCode.FATAL_ERROR), Response.Status.BAD_REQUEST, ErrorCode.FATAL_ERROR); - assertError(Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.APP_FATAL_ERROR), + assertError(List.of(ErrorCode.TRANSIENT_ERROR, ErrorCode.APP_FATAL_ERROR), Response.Status.BAD_REQUEST, ErrorCode.TRANSIENT_ERROR); - assertError(Arrays.asList(ErrorCode.FATAL_ERROR, ErrorCode.TRANSIENT_ERROR), + assertError(List.of(ErrorCode.FATAL_ERROR, ErrorCode.TRANSIENT_ERROR), Response.Status.BAD_REQUEST, ErrorCode.FATAL_ERROR, ErrorCode.TRANSIENT_ERROR); } diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 9554b517586..e2afa0e91f4 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -210,6 +210,18 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>model-integration</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-llama</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- TEST scope --> <dependency> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index e74fe22c588..5e66e1bb746 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -555,11 +555,15 @@ "public" ], "methods" : [ + "public void <init>(java.lang.String, boolean, java.lang.String, int, int, boolean)", "public void <init>(java.lang.String, boolean, java.lang.String, int, int)", "public void setMaxEditDistance(int)", "public void setPrefixLength(int)", "public int getPrefixLength()", "public int getMaxEditDistance()", + "public boolean isPrefixMatch()", + "public void setPrefixMatch(boolean)", + "protected boolean hasPrefixMatchSemantics()", "public void setValue(java.lang.String)", "public java.lang.String getRawWord()", "public boolean isWords()", @@ -820,6 +824,7 @@ "public abstract java.lang.String getName()", "public void setFilter(boolean)", "public boolean isFilter()", + "protected boolean hasPrefixMatchSemantics()", "public com.yahoo.prelude.query.Item$ItemCreator getCreator()", "public void setCreator(com.yahoo.prelude.query.Item$ItemCreator)", "public void setWeight(int)", @@ -7842,6 +7847,21 @@ "public static final int emptyDocsumsCode" ] }, + "com.yahoo.search.result.EventStream$ErrorEvent" : { + "superClass" : "com.yahoo.search.result.EventStream$Event", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(int, java.lang.String, com.yahoo.search.result.ErrorMessage)", + "public java.lang.String source()", + "public int code()", + "public java.lang.String message()", + "public com.yahoo.search.result.Hit asHit()" + ], + "fields" : [ ] + }, "com.yahoo.search.result.EventStream$Event" : { "superClass" : "com.yahoo.component.provider.ListenableFreezableClass", "interfaces" : [ @@ -9149,99 +9169,6 @@ ], "fields" : [ ] }, - "ai.vespa.llm.clients.ConfigurableLanguageModel" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "ai.vespa.llm.LanguageModel" - ], - "attributes" : [ - "public", - "abstract" - ], - "methods" : [ - "public void <init>()", - "public void <init>(ai.vespa.llm.clients.LlmClientConfig, com.yahoo.container.jdisc.secretstore.SecretStore)", - "protected java.lang.String getApiKey(ai.vespa.llm.InferenceParameters)", - "protected void setApiKey(ai.vespa.llm.InferenceParameters)", - "protected java.lang.String getEndpoint()", - "protected void setEndpoint(ai.vespa.llm.InferenceParameters)" - ], - "fields" : [ ] - }, - "ai.vespa.llm.clients.LlmClientConfig$Builder" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes" : [ - "public", - "final" - ], - "methods" : [ - "public void <init>()", - "public void <init>(ai.vespa.llm.clients.LlmClientConfig)", - "public ai.vespa.llm.clients.LlmClientConfig$Builder apiKeySecretName(java.lang.String)", - "public ai.vespa.llm.clients.LlmClientConfig$Builder endpoint(java.lang.String)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public ai.vespa.llm.clients.LlmClientConfig build()" - ], - "fields" : [ ] - }, - "ai.vespa.llm.clients.LlmClientConfig$Producer" : { - "superClass" : "java.lang.Object", - "interfaces" : [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes" : [ - "public", - "interface", - "abstract" - ], - "methods" : [ - "public abstract void getConfig(ai.vespa.llm.clients.LlmClientConfig$Builder)" - ], - "fields" : [ ] - }, - "ai.vespa.llm.clients.LlmClientConfig" : { - "superClass" : "com.yahoo.config.ConfigInstance", - "interfaces" : [ ], - "attributes" : [ - "public", - "final" - ], - "methods" : [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public void <init>(ai.vespa.llm.clients.LlmClientConfig$Builder)", - "public java.lang.String apiKeySecretName()", - "public java.lang.String endpoint()" - ], - "fields" : [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "ai.vespa.llm.clients.OpenAI" : { - "superClass" : "ai.vespa.llm.clients.ConfigurableLanguageModel", - "interfaces" : [ ], - "attributes" : [ - "public" - ], - "methods" : [ - "public void <init>(ai.vespa.llm.clients.LlmClientConfig, com.yahoo.container.jdisc.secretstore.SecretStore)", - "public java.util.List complete(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters)", - "public java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters, java.util.function.Consumer)" - ], - "fields" : [ ] - }, "ai.vespa.search.llm.LLMSearcher" : { "superClass" : "com.yahoo.search.Searcher", "interfaces" : [ ], diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index 860fc69af91..f565315b775 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; import java.util.List; +import java.util.concurrent.RejectedExecutionException; import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -83,27 +84,41 @@ public class LLMSearcher extends Searcher { protected Result complete(Query query, Prompt prompt) { var options = new InferenceParameters(getApiKeyHeader(query), s -> lookupProperty(s, query)); var stream = lookupPropertyBool(STREAM_PROPERTY, query, this.stream); // query value overwrites config - return stream ? completeAsync(query, prompt, options) : completeSync(query, prompt, options); + try { + return stream ? completeAsync(query, prompt, options) : completeSync(query, prompt, options); + } catch (RejectedExecutionException e) { + return new Result(query, new ErrorMessage(429, e.getMessage())); + } + } + + private boolean shouldAddPrompt(Query query) { + return query.getTrace().getLevel() >= 1; + } + + private boolean shouldAddTokenStats(Query query) { + return query.getTrace().getLevel() >= 1; } private Result completeAsync(Query query, Prompt prompt, InferenceParameters options) { - EventStream eventStream = new EventStream(); + final EventStream eventStream = new EventStream(); - if (query.getTrace().getLevel() >= 1) { + if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } - languageModel.completeAsync(prompt, options, token -> { - eventStream.add(token.text()); + final TokenStats tokenStats = new TokenStats(); + languageModel.completeAsync(prompt, options, completion -> { + tokenStats.onToken(); + handleCompletion(eventStream, completion); }).exceptionally(exception -> { - int errorCode = 400; - if (exception instanceof LanguageModelException languageModelException) { - errorCode = languageModelException.code(); - } - eventStream.error(languageModelId, new ErrorMessage(errorCode, exception.getMessage())); + handleException(eventStream, exception); eventStream.markComplete(); return Completion.FinishReason.error; }).thenAccept(finishReason -> { + tokenStats.onCompletion(); + if (shouldAddTokenStats(query)) { + eventStream.add(tokenStats.report(), "stats"); + } eventStream.markComplete(); }); @@ -112,10 +127,26 @@ public class LLMSearcher extends Searcher { return new Result(query, hitGroup); } + private void handleCompletion(EventStream eventStream, Completion completion) { + if (completion.finishReason() == Completion.FinishReason.error) { + eventStream.add(completion.text(), "error"); + } else { + eventStream.add(completion.text()); + } + } + + private void handleException(EventStream eventStream, Throwable exception) { + int errorCode = 400; + if (exception instanceof LanguageModelException languageModelException) { + errorCode = languageModelException.code(); + } + eventStream.error(languageModelId, new ErrorMessage(errorCode, exception.getMessage())); + } + private Result completeSync(Query query, Prompt prompt, InferenceParameters options) { EventStream eventStream = new EventStream(); - if (query.getTrace().getLevel() >= 1) { + if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } @@ -169,4 +200,35 @@ public class LLMSearcher extends Searcher { return lookupPropertyWithOrWithoutPrefix(API_KEY_HEADER, p -> query.getHttpRequest().getHeader(p)); } + private static class TokenStats { + + private long start; + private long timeToFirstToken; + private long timeToLastToken; + private long tokens = 0; + + TokenStats() { + start = System.currentTimeMillis(); + } + + void onToken() { + if (tokens == 0) { + timeToFirstToken = System.currentTimeMillis() - start; + } + tokens++; + } + + void onCompletion() { + timeToLastToken = System.currentTimeMillis() - start; + } + + String report() { + return "Time to first token: " + timeToFirstToken + " ms, " + + "Generation time: " + timeToLastToken + " ms, " + + "Generated tokens: " + tokens + " " + + String.format("(%.2f tokens/sec)", tokens / (timeToLastToken / 1000.0)); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index b5e5ed9ed8f..7d4e2e8ef0a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -13,7 +13,6 @@ import com.yahoo.data.access.Inspector; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -62,7 +61,7 @@ public class FastHit extends Hit { * that most fields passes through the container with no processing most * of the time. */ - private List<SummaryData> summaries = Collections.emptyList(); + private List<SummaryData> summaries = List.of(); /** Removed field values, which should therefore not be returned if present in summary data */ private Set<String> removedFields = null; @@ -275,7 +274,8 @@ public class FastHit extends Hit { /** Removes all fields of this */ @Override public void clearFields() { - summaries.clear(); + if (!summaries.isEmpty()) + summaries.clear(); if (removedFields != null) removedFields = null; super.clearFields(); @@ -481,7 +481,7 @@ public class FastHit extends Hit { private Set<String> createSet() { if (this.fieldSet != null) return this.fieldSet; - if ( ! hit.hasFields() && hit.summaries.isEmpty()) return Collections.emptySet(); // shortcut + if ( ! hit.hasFields() && hit.summaries.isEmpty()) return Set.of(); // shortcut Set<String> fields = new HashSet<>(); if (hit.hasFields()) diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java index 3cf86a70985..b900dee20ba 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java @@ -16,15 +16,21 @@ public class FuzzyItem extends TermItem { private int maxEditDistance; private int prefixLength; + private boolean prefixMatch; public static int DEFAULT_MAX_EDIT_DISTANCE = 2; public static int DEFAULT_PREFIX_LENGTH = 0; - public FuzzyItem(String indexName, boolean isFromQuery, String term, int maxEditDistance, int prefixLength) { + public FuzzyItem(String indexName, boolean isFromQuery, String term, int maxEditDistance, int prefixLength, boolean prefixMatch) { super(indexName, isFromQuery, null); setValue(term); setMaxEditDistance(maxEditDistance); setPrefixLength(prefixLength); + setPrefixMatch(prefixMatch); + } + + public FuzzyItem(String indexName, boolean isFromQuery, String term, int maxEditDistance, int prefixLength) { + this(indexName, isFromQuery, term, maxEditDistance, prefixLength, false); } public void setMaxEditDistance(int maxEditDistance) { @@ -43,6 +49,19 @@ public class FuzzyItem extends TermItem { return this.maxEditDistance; } + public boolean isPrefixMatch() { + return this.prefixMatch; + } + + public void setPrefixMatch(boolean prefixMatch) { + this.prefixMatch = prefixMatch; + } + + @Override + protected boolean hasPrefixMatchSemantics() { + return this.prefixMatch; + } + @Override public void setValue(String value) { this.term = value; @@ -89,43 +108,39 @@ public class FuzzyItem extends TermItem { } @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!super.equals(obj)) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - FuzzyItem other = (FuzzyItem) obj; - if (!this.term.equals(other.term)) return false; - if (this.maxEditDistance != other.maxEditDistance) return false; - if (this.prefixLength != other.prefixLength) return false; - return true; + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + FuzzyItem fuzzyItem = (FuzzyItem) o; + return maxEditDistance == fuzzyItem.maxEditDistance && + prefixLength == fuzzyItem.prefixLength && + prefixMatch == fuzzyItem.prefixMatch && + Objects.equals(term, fuzzyItem.term); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), term, maxEditDistance, prefixLength); + return Objects.hash(super.hashCode(), term, maxEditDistance, prefixLength, prefixMatch); } @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); - buffer.append("("); + buffer.append('('); buffer.append(this.term); - buffer.append(","); + buffer.append(','); buffer.append(this.maxEditDistance); - buffer.append(","); + buffer.append(','); buffer.append(this.prefixLength); - buffer.append(")"); - buffer.append(" "); + buffer.append(','); + buffer.append(this.prefixMatch); + buffer.append(") "); } @Override protected void encodeThis(ByteBuffer buffer) { + // Prefix matching is communicated via term header flags super.encodeThis(buffer); putString(getIndexedString(), buffer); IntegerCompressor.putCompressedPositiveNumber(this.maxEditDistance, buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index f43b55424e6..099c546e3f0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -161,6 +161,16 @@ public abstract class Item implements Cloneable { } /** + * Indicates that a query item that does not normally match with prefix semantics + * should do so for this particular query item instance. + * + * False by default; should be overridden by subclasses that want to signal this behavior. + */ + protected boolean hasPrefixMatchSemantics() { + return false; + } + + /** * Returns the item creator value. * * @deprecated use isFilter(boolean) @@ -286,6 +296,7 @@ public abstract class Item implements Cloneable { byte FLAGS_SPECIALTOKEN = 0x02; byte FLAGS_NOPOSITIONDATA = 0x04; byte FLAGS_ISFILTER = 0x08; + byte FLAGS_PREFIX_MATCH = 0x10; byte ret = 0; if (!isRanked()) { @@ -300,6 +311,9 @@ public abstract class Item implements Cloneable { if (isFilter()) { ret |= FLAGS_ISFILTER; } + if (hasPrefixMatchSemantics()) { + ret |= FLAGS_PREFIX_MATCH; + } return ret; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java index 6e1a2be684a..e7bdf640661 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java @@ -46,7 +46,7 @@ public abstract class SimpleIndexedItem extends SimpleTaggableItem implements In /** Appends the index prefix if necessary */ protected void appendIndexString(StringBuilder buffer) { - if (!getIndexName().equals("")) { + if (!getIndexName().isEmpty()) { buffer.append(getIndexName()); buffer.append(":"); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java index 44c0cb45732..520879be928 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java @@ -6,7 +6,6 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Item; import com.yahoo.search.query.parser.Parser; -import java.util.Collections; import java.util.Set; /** @@ -23,7 +22,7 @@ public interface CustomParser extends Parser { Set<String> toSearch, IndexFacts indexFacts, String defaultIndexName) { if (indexFacts == null) indexFacts = new IndexFacts(); - return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Collections.emptySet()), defaultIndexName); + return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Set.of()), defaultIndexName); } Item parse(String queryToParse, String filterToParse, Language parsingLanguage, diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 1c0d3ea2ee9..5a42efaaea0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -8,8 +8,8 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; -import java.util.Collections; import java.util.List; +import java.util.Set; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -63,7 +63,7 @@ public final class Tokenizer { * @return a read-only list of tokens. This list can only be used by this thread */ public List<Token> tokenize(String string) { - return tokenize(string, new IndexFacts().newSession(Collections.emptySet(), Collections.emptySet())); + return tokenize(string, new IndexFacts().newSession(Set.of(), Set.of())); } /** diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index ef453492a5d..e2e0ea90632 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -14,7 +14,6 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -46,7 +45,7 @@ public class SemanticSearcher extends Searcher { } public SemanticSearcher(RuleBase ... ruleBases) { - this(Arrays.asList(ruleBases)); + this(List.of(ruleBases)); } @Inject diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 3227b047984..4ec3fa358d2 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -772,7 +772,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private String serializeSortingAndLimits(boolean includeHitsAndOffset) { StringBuilder insert = new StringBuilder(); - if (getRanking().getSorting() != null && getRanking().getSorting().fieldOrders().size() > 0) { + if (getRanking().getSorting() != null && !getRanking().getSorting().fieldOrders().isEmpty()) { serializeSorting(insert); } if (includeHitsAndOffset) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java index 88d6b68a610..9c41c0b930f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerResult.java @@ -7,7 +7,6 @@ import com.yahoo.search.Result; import com.yahoo.search.query.Sorting; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -22,7 +21,7 @@ public class InvokerResult { public InvokerResult(Result result) { this.result = result; - this.leanHits = Collections.emptyList(); + this.leanHits = List.of(); } public InvokerResult(Query query, int expectedHits) { @@ -54,7 +53,8 @@ public class InvokerResult { fh.setCached(false); result.hits().add(fh); } - leanHits.clear(); + if (!leanHits.isEmpty()) + leanHits.clear(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 9302ce5b7c6..add562bd5ad 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -8,7 +8,6 @@ import com.yahoo.search.dispatch.searchcluster.Node; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -163,7 +162,7 @@ public class SearchPath { nodes = parseNodeRange(nodes, ret); } else { if (isWildcard(nodes)) { // any node will be accepted - return Collections.emptyList(); + return List.of(); } nodes = parseNodeNum(nodes, ret); } @@ -231,7 +230,7 @@ public class SearchPath { public Collection<Integer> matches(int max) { if (from >= max) { - return Collections.emptyList(); + return List.of(); } int end = Math.min(to, max); return IntStream.range(from, end).boxed().toList(); diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 72184c5ea32..4d9111b2711 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -9,8 +9,6 @@ import com.yahoo.component.chain.Chain; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.errorhandling.Results; -import com.yahoo.errorhandling.Results.Builder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -19,12 +17,12 @@ import com.yahoo.search.Searcher; import com.yahoo.search.federation.selection.FederationTarget; import com.yahoo.search.federation.selection.TargetSelector; import com.yahoo.search.federation.sourceref.ModifyQueryAndResult; +import com.yahoo.search.federation.sourceref.ResolveResult; import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; import com.yahoo.search.federation.sourceref.SearchChainResolver; import com.yahoo.search.federation.sourceref.SingleTarget; import com.yahoo.search.federation.sourceref.SourceRefResolver; import com.yahoo.search.federation.sourceref.SourcesTarget; -import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; import com.yahoo.search.federation.sourceref.VirtualSourceResolver; import com.yahoo.search.query.Properties; import com.yahoo.search.result.ErrorMessage; @@ -53,6 +51,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -206,14 +205,44 @@ public class FederationSearcher extends ForkingSearcher { setRequestTimeoutInMilliseconds(searchChain.requestTimeoutMillis()); } + private static List<String> extractErrors(List<ResolveResult> results) { + List<String> errors = List.of(); + for (ResolveResult result : results) { + if (result.errorMsg() != null) { + if (errors.isEmpty()) { + errors = new ArrayList<>(); + } + errors.add(result.errorMsg()); + } + } + return errors; + } + + private static List<SearchChainInvocationSpec> extractSpecs(List<ResolveResult> results) { + List<SearchChainInvocationSpec> errors = List.of(); + for (ResolveResult result : results) { + if (result.invocationSpec() != null) { + if (errors.isEmpty()) { + errors = List.of(result.invocationSpec()); + } else if (errors.size() == 1) { + errors = new ArrayList<>(errors); + errors.add(result.invocationSpec()); + } else { + errors.add(result.invocationSpec()); + } + } + } + return errors; + } + @Override public Result search(Query query, Execution execution) { Result mergedResults = execution.search(query); var targets = getTargets(query.getModel().getSources(), query.properties()); - warnIfUnresolvedSearchChains(targets.errors(), mergedResults.hits()); + warnIfUnresolvedSearchChains(extractErrors(targets), mergedResults.hits()); - var prunedTargets = pruneTargetsWithoutDocumentTypes(query.getModel().getRestrict(), targets.data()); + var prunedTargets = pruneTargetsWithoutDocumentTypes(query.getModel().getRestrict(), extractSpecs(targets)); var regularTargetHandlers = resolveSearchChains(prunedTargets, execution.searchChainRegistry()); query.errors().addAll(regularTargetHandlers.errors()); @@ -311,32 +340,19 @@ public class FederationSearcher extends ForkingSearcher { .forEach((k, v) -> outgoing.properties().set(k, v)); } - private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { - String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) + + private ErrorMessage missingSearchChainsErrorMessage(List<String> errors) { + String message = String.join(" ", new TreeSet<>(errors)) + " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; return ErrorMessage.createInvalidQueryParameter(message); } private List<String> allSourceRefDescriptions() { - List<String> descriptions = new ArrayList<>(); - - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) - descriptions.add(target.searchRefDescription()); - return descriptions; - } - - private static Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { - Set<String> messages = new LinkedHashSet<>(); - for (UnresolvedSearchChainException exception : unresolvedSearchChainExceptions) { - messages.add(exception.getMessage()); - } - return messages; + return searchChainResolver.allTopLevelTargets().stream().map(com.yahoo.search.federation.sourceref.Target::searchRefDescription).toList(); } - private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets, - HitGroup errorHitGroup) { - if (!missingTargets.isEmpty()) { - errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets)); + private void warnIfUnresolvedSearchChains(List<String> errorMessages, HitGroup errorHitGroup) { + if (!errorMessages.isEmpty()) { + errorHitGroup.addError(missingSearchChainsErrorMessage(errorMessages)); } } @@ -344,7 +360,7 @@ public class FederationSearcher extends ForkingSearcher { public Collection<CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry) { List<CommentedSearchChain> searchChains = new ArrayList<>(); - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.allTopLevelTargets()) { + for (var target : searchChainResolver.allTopLevelTargets()) { if (target instanceof SourcesTarget) { searchChains.addAll(commentedSourceProviderSearchChains((SourcesTarget)target, registry)); } else if (target instanceof SingleTarget) { @@ -468,40 +484,32 @@ public class FederationSearcher extends ForkingSearcher { return orderer; } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties) { + private List<ResolveResult> getTargets(Set<String> sources, Properties properties) { return sources.isEmpty() ? defaultSearchChains(properties): resolveSources(sources, properties); } - private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sourcesInQuery, Properties properties) { - Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + private List<ResolveResult> resolveSources(Set<String> sourcesInQuery, Properties properties) { + List<ResolveResult> result = new ArrayList<>(); Set<String> sources = virtualSourceResolver.resolve(sourcesInQuery); for (String source : sources) { - try { - result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties)); - } catch (UnresolvedSearchChainException e) { - result.addError(e); - } + result.addAll(sourceRefResolver.resolve(asSourceSpec(source), properties)); } - return result.build(); + return List.copyOf(result); } - public Results<SearchChainInvocationSpec, UnresolvedSearchChainException> defaultSearchChains(Properties sourceToProviderMap) { - Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>(); + public List<ResolveResult> defaultSearchChains(Properties sourceToProviderMap) { + List<ResolveResult> result = new ArrayList<>(); - for (com.yahoo.search.federation.sourceref.Target target : searchChainResolver.defaultTargets()) { - try { - result.addData(target.responsibleSearchChain(sourceToProviderMap)); - } catch (UnresolvedSearchChainException e) { - result.addError(e); - } + for (var target : searchChainResolver.defaultTargets()) { + result.add(target.responsibleSearchChain(sourceToProviderMap)); } - return result.build(); + return List.copyOf(result); } diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java b/container-search/src/main/java/com/yahoo/search/federation/Results.java index 939d2276efc..7598a14f759 100644 --- a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java +++ b/container-search/src/main/java/com/yahoo/search/federation/Results.java @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.errorhandling; +package com.yahoo.search.federation; import java.util.ArrayList; import java.util.Collection; @@ -13,15 +13,11 @@ public class Results<DATA, ERROR> { private final List<DATA> data; private final List<ERROR> errors; - public Results(List<DATA> data, List<ERROR> errors) { + private Results(List<DATA> data, List<ERROR> errors) { this.data = List.copyOf(data); this.errors = List.copyOf(errors); } - public boolean hasErrors() { - return !errors.isEmpty(); - } - public List<DATA> data() { return data; } @@ -37,19 +33,10 @@ public class Results<DATA, ERROR> { public void addData(DATA d) { data.add(d); } - - public void addAllData(Collection<? extends DATA> d) { - data.addAll(d); - } - public void addError(ERROR e) { errors.add(e); } - public void addAllErrors(Collection<? extends ERROR> e) { - errors.addAll(e); - } - public Results<DATA, ERROR> build() { return new Results<>(data, errors); } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java new file mode 100644 index 00000000000..d9681140ae9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/ResolveResult.java @@ -0,0 +1,14 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref; + +/** + * @author baldersheim + */ +public record ResolveResult(SearchChainInvocationSpec invocationSpec, String errorMsg) { + ResolveResult(SearchChainInvocationSpec invocationSpec) { + this(invocationSpec, null); + } + ResolveResult(String errorMsg) { + this(null, errorMsg); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java index 7dc65c819e4..9e45b6576a6 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java @@ -46,16 +46,6 @@ public class SearchChainResolver { public static class Builder { - public interface InvocationSpecFactory { - SearchChainInvocationSpec create(ComponentId searchChainId, FederationOptions federationOptions, List<String> schemas); - } - - private class DefaultInvocationSpecFactory implements InvocationSpecFactory { - public SearchChainInvocationSpec create(ComponentId searchChainId, FederationOptions federationOptions, List<String> schemas) { - return new SearchChainInvocationSpec(searchChainId, federationOptions, schemas); - } - } - private final SortedSet<Target> defaultTargets = new TreeSet<>(); private final ComponentRegistry<Target> targets = new ComponentRegistry<>() { @@ -137,19 +127,13 @@ public class SearchChainResolver { this.defaultTargets = Collections.unmodifiableSortedSet(defaultTargets); } - public SearchChainInvocationSpec resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) - throws UnresolvedSearchChainException { + public ResolveResult resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) { - Target target = resolveTarget(sourceRef); - return target.responsibleSearchChain(sourceToProviderMap); - } - - private Target resolveTarget(ComponentSpecification sourceRef) throws UnresolvedSearchChainException { Target target = targets.getComponent(sourceRef); if (target == null) { - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); + return new ResolveResult(SourceRefResolver.createForMissingSourceRef(sourceRef)); } - return target; + return target.responsibleSearchChain(sourceToProviderMap); } public SortedSet<Target> allTopLevelTargets() { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java index 608566552cd..3de67908217 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java @@ -17,8 +17,8 @@ public class SingleTarget extends Target { } @Override - public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) { - return searchChainInvocationSpec; + public ResolveResult responsibleSearchChain(Properties queryProperties) { + return new ResolveResult(searchChainInvocationSpec); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java index 2e7849dd85a..b5c40db01f8 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java @@ -4,10 +4,9 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentSpecification; import com.yahoo.processing.request.Properties; -import java.util.LinkedHashSet; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; /** * Maps a source reference to search chain invocation specs. @@ -24,21 +23,18 @@ public class SourceRefResolver { this.schema2Clusters = schema2Clusters; } - public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, - Properties sourceToProviderMap) throws UnresolvedSearchChainException { - try { - return Set.of(searchChainResolver.resolve(sourceRef, sourceToProviderMap)); - } catch (UnresolvedSourceRefException e) { + public List<ResolveResult> resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap) { + ResolveResult searchChainResolveResult = searchChainResolver.resolve(sourceRef, sourceToProviderMap); + if (searchChainResolveResult.invocationSpec() == null) { return resolveClustersWithDocument(sourceRef, sourceToProviderMap); } + return List.of(searchChainResolveResult); } - private Set<SearchChainInvocationSpec> resolveClustersWithDocument(ComponentSpecification sourceRef, - Properties sourceToProviderMap) - throws UnresolvedSearchChainException { + private List<ResolveResult> resolveClustersWithDocument(ComponentSpecification sourceRef, Properties sourceToProviderMap) { if (hasOnlyName(sourceRef)) { - Set<SearchChainInvocationSpec> clusterSearchChains = new LinkedHashSet<>(); + List<ResolveResult> clusterSearchChains = new ArrayList<>(); List<String> clusters = schema2Clusters.getOrDefault(sourceRef.getName(), List.of()); for (String cluster : clusters) { @@ -48,21 +44,22 @@ public class SourceRefResolver { if ( ! clusterSearchChains.isEmpty()) return clusterSearchChains; } - throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef); + return List.of(new ResolveResult(createForMissingSourceRef(sourceRef))); } - private SearchChainInvocationSpec resolveClusterSearchChain(String cluster, - ComponentSpecification sourceRef, - Properties sourceToProviderMap) - throws UnresolvedSearchChainException { - try { - return searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap); - } - catch (UnresolvedSearchChainException e) { - throw new UnresolvedSearchChainException("Failed to resolve cluster search chain '" + cluster + - "' when using source ref '" + sourceRef + - "' as a document name."); + static String createForMissingSourceRef(ComponentSpecification source) { + return "Could not resolve source ref '" + source + "'."; + } + + private ResolveResult resolveClusterSearchChain(String cluster, + ComponentSpecification sourceRef, + Properties sourceToProviderMap) { + var resolveResult = searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap); + if (resolveResult.invocationSpec() == null) { + return new ResolveResult("Failed to resolve cluster search chain '" + cluster + + "' when using source ref '" + sourceRef + "' as a document name."); } + return resolveResult; } private boolean hasOnlyName(ComponentSpecification sourceSpec) { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java index b6d99758c7b..a3c0328290d 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java @@ -16,7 +16,7 @@ import java.util.TreeSet; public class SourcesTarget extends Target { - private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {}; + private final ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources = new ComponentRegistry<>() {}; private SearchChainInvocationSpec defaultProviderSource; @@ -25,10 +25,10 @@ public class SourcesTarget extends Target { } @Override - public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException { + public ResolveResult responsibleSearchChain(Properties queryProperties) { ComponentSpecification providerSpecification = providerSpecificationForSource(queryProperties); if (providerSpecification == null) { - return defaultProviderSource; + return new ResolveResult(defaultProviderSource); } else { return lookupProviderSource(providerSpecification); } @@ -36,11 +36,7 @@ public class SourcesTarget extends Target { @Override public String searchRefDescription() { - StringBuilder builder = new StringBuilder(sourceId().stringValue()); - builder.append("[provider = "). - append(Joiner.on(", ").join(allProviderIdsStringValue())). - append("]"); - return builder.toString(); + return sourceId().stringValue() + "[provider = " + Joiner.on(", ").join(allProviderIdsStringValue()) + "]"; } private SortedSet<String> allProviderIdsStringValue() { @@ -51,14 +47,13 @@ public class SourcesTarget extends Target { return result; } - private SearchChainInvocationSpec lookupProviderSource(ComponentSpecification providerSpecification) - throws UnresolvedSearchChainException { + private ResolveResult lookupProviderSource(ComponentSpecification providerSpecification) { ComponentAdaptor<SearchChainInvocationSpec> providerSource = providerSources.getComponent(providerSpecification); if (providerSource == null) - throw UnresolvedProviderException.createForMissingProvider(sourceId(), providerSpecification); + return new ResolveResult("No provider '" + sourceId() + "' for source '" + providerSpecification + "'."); - return providerSource.model; + return new ResolveResult(providerSource.model); } public void freeze() { diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java index 38baf084d97..d35f7f7b181 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java +++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java @@ -23,9 +23,8 @@ public abstract class Target extends AbstractComponent { this(localId, false); } - public abstract SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException; + public abstract ResolveResult responsibleSearchChain(Properties queryProperties); public abstract String searchRefDescription(); abstract void freeze(); - } diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java deleted file mode 100644 index aa21ad3b369..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; - -/** - * @author Tony Vaagenes - */ -@SuppressWarnings("serial") -class UnresolvedProviderException extends UnresolvedSearchChainException { - UnresolvedProviderException(String msg) { - super(msg); - } - - static UnresolvedSearchChainException createForMissingProvider(ComponentId source, - ComponentSpecification provider) { - return new UnresolvedProviderException("No provider '" + provider + "' for source '" + source + "'."); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java deleted file mode 100644 index 0c8562e6032..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -/** - * Thrown if a search chain can not be resolved from one or more ids. - * @author Tony Vaagenes - */ -public class UnresolvedSearchChainException extends Exception { - public UnresolvedSearchChainException(String msg) { - super(msg); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java deleted file mode 100644 index fa2c1da13f0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.sourceref; - -import com.yahoo.component.ComponentSpecification; - -/** - * @author Tony Vaagenes - */ -class UnresolvedSourceRefException extends UnresolvedSearchChainException { - UnresolvedSourceRefException(String msg) { - super(msg); - } - - - static UnresolvedSearchChainException createForMissingSourceRef(ComponentSpecification source) { - return new UnresolvedSourceRefException("Could not resolve source ref '" + source + "'."); - } -} diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java index 3ad610f6ee0..da0b258d6ef 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java @@ -15,7 +15,6 @@ import com.yahoo.search.query.Select; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -83,7 +82,7 @@ public class GroupingQueryParser extends Searcher { private List<Continuation> getContinuations(String param) { if (param == null) { - return Collections.emptyList(); + return List.of(); } List<Continuation> ret = new LinkedList<>(); for (String str : param.split(" ")) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index 6d2b416700a..e319740f741 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -27,7 +27,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.logging.Logger; @@ -188,7 +187,7 @@ public class UniqueGroupingSearcher extends Searcher { private static List<Hit> getRequestedHits(GroupList resultGroups, int offset, int hits) { List<Hit> receivedHits = getAllHitsFromGroupingResult(resultGroups); if (receivedHits.size() <= offset) { - return Collections.emptyList(); // There weren't any hits as far out as requested. + return List.of(); // There weren't any hits as far out as requested. } int lastRequestedHit = Math.min(offset + hits, receivedHits.size()); return receivedHits.subList(offset, lastRequestedHit); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java index 3c152f00c66..f911b695853 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; -import java.util.stream.Collectors; +import java.util.List; /** * This class represents a timestamp-formatter function in a {@link GroupingExpression}. It evaluates to a string on the @@ -23,14 +22,12 @@ public class DateFunction extends FunctionNode { } private DateFunction(String label, Integer level, GroupingExpression exp) { - super("time.date", label, level, Arrays.asList(exp)); + super("time.date", label, level, List.of(exp)); } @Override public DateFunction copy() { - return new DateFunction(getLabel(), - getLevelOrNull(), - getArg(0).copy()); + return new DateFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java index c9639b4ea71..654e969221b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; -import java.util.stream.Collectors; +import java.util.List; /** * This class represents a day-of-month timestamp-function in a {@link GroupingExpression}. It evaluates to a long that @@ -23,14 +22,12 @@ public class DayOfMonthFunction extends FunctionNode { } private DayOfMonthFunction(String label, Integer level, GroupingExpression exp) { - super("time.dayofmonth", label, level, Arrays.asList(exp)); + super("time.dayofmonth", label, level, List.of(exp)); } @Override public DayOfMonthFunction copy() { - return new DayOfMonthFunction(getLabel(), - getLevelOrNull(), - getArg(0).copy()); + return new DayOfMonthFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java index 644f16a4d17..7b91ab34115 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a day-of-week timestamp-function in a {@link GroupingExpression}. It evaluates to a long that @@ -22,14 +22,12 @@ public class DayOfWeekFunction extends FunctionNode { } private DayOfWeekFunction(String label, Integer level, GroupingExpression exp) { - super("time.dayofweek", label, level, Arrays.asList(exp)); + super("time.dayofweek", label, level, List.of(exp)); } @Override public DayOfWeekFunction copy() { - return new DayOfWeekFunction(getLabel(), - getLevelOrNull(), - getArg(0).copy()); + return new DayOfWeekFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java index f45a7724794..9f7ff969e49 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a day-of-year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that @@ -22,14 +22,12 @@ public class DayOfYearFunction extends FunctionNode { } private DayOfYearFunction(String label, Integer level, GroupingExpression exp) { - super("time.dayofyear", label, level, Arrays.asList(exp)); + super("time.dayofyear", label, level, List.of(exp)); } @Override public DayOfYearFunction copy() { - return new DayOfYearFunction(getLabel(), - getLevelOrNull(), - getArg(0).copy()); + return new DayOfYearFunction(getLabel(), getLevelOrNull(), getArg(0).copy()); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java index 482d6fe93ce..1477d125847 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; -import java.util.stream.Collectors; +import java.util.List; /** * This class represents debug_wait function in a {@link GroupingExpression}. For each hit evaluated, @@ -25,7 +24,7 @@ public class DebugWaitFunction extends FunctionNode { } private DebugWaitFunction(String label, Integer level, GroupingExpression arg1, DoubleValue arg2, BooleanValue arg3) { - super("debugwait", label, level, Arrays.asList(arg1, arg2, arg3)); + super("debugwait", label, level, List.of(arg1, arg2, arg3)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java index 04f6d155699..6e0b721a584 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java @@ -1,7 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; + +import java.util.List; /** * This class represents a fixed-width bucket-function in a {@link GroupingExpression}. It maps the input into the given @@ -23,7 +24,7 @@ public class FixedWidthFunction extends FunctionNode { } private FixedWidthFunction(String label, Integer level, GroupingExpression exp, ConstantValue width) { - super("fixedwidth", label, level, Arrays.asList(exp, width)); + super("fixedwidth", label, level, List.of(exp, width)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java index c1d86ed3059..d2be51c074b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java @@ -66,12 +66,12 @@ public abstract class FunctionNode extends GroupingExpression implements Iterabl @SuppressWarnings("unchecked") protected static <T> List<T> asList(T arg1, T... argN) { - return asList(Arrays.asList(arg1), Arrays.asList(argN)); + return asList(List.of(arg1), List.of(argN)); } @SuppressWarnings("unchecked") protected static <T> List<T> asList(T arg1, T arg2, T... argN) { - return asList(Arrays.asList(arg1, arg2), Arrays.asList(argN)); + return asList(List.of(arg1, arg2), List.of(argN)); } protected static <T> List<T> asList(List<T> foo, List<T> bar) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java index b6fc0ffb968..fe49b46fc6f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java @@ -537,12 +537,7 @@ public abstract class GroupingOperation extends GroupingNode { } else if (level == 1) { return "single group"; } else { - StringBuilder ret = new StringBuilder(); - for (int i = 1; i < level; ++i) { - ret.append("list of "); - } - ret.append("groups"); - return ret.toString(); + return "list of ".repeat(level - 1) + "groups"; } } @@ -571,8 +566,8 @@ public abstract class GroupingOperation extends GroupingNode { * @throws IllegalArgumentException thrown if the string could not be parsed */ public static List<GroupingOperation> fromStringAsList(String string) { - if (string == null || string.trim().length() == 0) { - return Collections.emptyList(); + if (string == null || string.trim().isEmpty()) { + return List.of(); } GroupingParserInput input = new GroupingParserInput(string); try { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java index b65e1bc2405..95dcf608a95 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents an hour-of-day timestamp-function in a {@link GroupingExpression}. It evaluates to a long that @@ -22,7 +22,7 @@ public class HourOfDayFunction extends FunctionNode { } private HourOfDayFunction(String label, Integer level, GroupingExpression exp) { - super("time.hourofday", label, level, Arrays.asList(exp)); + super("time.hourofday", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java index 70f3515371b..8b6ffc53680 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathACosFunction extends FunctionNode { } private MathACosFunction(String label, Integer level, GroupingExpression exp) { - super("math.acos", label, level, Arrays.asList(exp)); + super("math.acos", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java index 6b31e1cfaff..34570aedc58 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathACosHFunction extends FunctionNode { } private MathACosHFunction(String label, Integer level, GroupingExpression exp) { - super("math.acosh", label, level, Arrays.asList(exp)); + super("math.acosh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java index cbb99536af1..1fb0b89d0aa 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathASinFunction extends FunctionNode { } private MathASinFunction(String label, Integer level, GroupingExpression exp) { - super("math.asin", label, level, Arrays.asList(exp)); + super("math.asin", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java index 9e159abe90f..e0d290c59dc 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathASinHFunction extends FunctionNode { } private MathASinHFunction(String label, Integer level, GroupingExpression exp) { - super("math.asinh", label, level, Arrays.asList(exp)); + super("math.asinh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java index a7302769853..50c40a2ad26 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathATanFunction extends FunctionNode { } private MathATanFunction(String label, Integer level, GroupingExpression exp) { - super("math.atan", label, level, Arrays.asList(exp)); + super("math.atan", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java index f33b8883a79..c8b34e36aaa 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathATanHFunction extends FunctionNode { } private MathATanHFunction(String label, Integer level, GroupingExpression exp) { - super("math.atanh", label, level, Arrays.asList(exp)); + super("math.atanh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java index 5e4ad8fb20f..ccec1752823 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathCbrtFunction extends FunctionNode { } private MathCbrtFunction(String label, Integer level, GroupingExpression exp) { - super("math.cbrt", label, level, Arrays.asList(exp)); + super("math.cbrt", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java index bbf303e0bce..56aaa16c2ae 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathCosFunction extends FunctionNode { } private MathCosFunction(String label, Integer level, GroupingExpression exp) { - super("math.cos", label, level, Arrays.asList(exp)); + super("math.cos", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java index da92164aec5..8d21435ca52 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathCosHFunction extends FunctionNode { } private MathCosHFunction(String label, Integer level, GroupingExpression exp) { - super("math.cosh", label, level, Arrays.asList(exp)); + super("math.cosh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java index 124b4fdaf40..625c6358081 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathExpFunction extends FunctionNode { } private MathExpFunction(String label, Integer level, GroupingExpression exp) { - super("math.exp", label, level, Arrays.asList(exp)); + super("math.exp", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java index cab480e8ff6..4c775a86b19 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * Represents the math.floor(expression) function @@ -21,7 +21,7 @@ public class MathFloorFunction extends FunctionNode { } private MathFloorFunction(String label, Integer level, GroupingExpression exp) { - super("math.floor", label, level, Arrays.asList(exp)); + super("math.floor", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java index ec2cb62b062..d76c464a6de 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java @@ -42,29 +42,28 @@ public abstract class MathFunctions { } } public static FunctionNode newInstance(Function type, GroupingExpression x, GroupingExpression y) { - switch (type) { - case EXP: return new MathExpFunction(x); - case POW: return new MathPowFunction(x, y); - case LOG: return new MathLogFunction(x); - case LOG1P: return new MathLog1pFunction(x); - case LOG10: return new MathLog10Function(x); - case SIN: return new MathSinFunction(x); - case ASIN: return new MathASinFunction(x); - case COS: return new MathCosFunction(x); - case ACOS: return new MathACosFunction(x); - case TAN: return new MathTanFunction(x); - case ATAN: return new MathATanFunction(x); - case SQRT: return new MathSqrtFunction(x); - case SINH: return new MathSinHFunction(x); - case ASINH: return new MathASinHFunction(x); - case COSH: return new MathCosHFunction(x); - case ACOSH: return new MathACosHFunction(x); - case TANH: return new MathTanHFunction(x); - case ATANH: return new MathATanHFunction(x); - case CBRT: return new MathCbrtFunction(x); - case HYPOT: return new MathHypotFunction(x, y); - case FLOOR: return new MathFloorFunction(x); - } - return null; + return switch (type) { + case EXP -> new MathExpFunction(x); + case POW -> new MathPowFunction(x, y); + case LOG -> new MathLogFunction(x); + case LOG1P -> new MathLog1pFunction(x); + case LOG10 -> new MathLog10Function(x); + case SIN -> new MathSinFunction(x); + case ASIN -> new MathASinFunction(x); + case COS -> new MathCosFunction(x); + case ACOS -> new MathACosFunction(x); + case TAN -> new MathTanFunction(x); + case ATAN -> new MathATanFunction(x); + case SQRT -> new MathSqrtFunction(x); + case SINH -> new MathSinHFunction(x); + case ASINH -> new MathASinHFunction(x); + case COSH -> new MathCosHFunction(x); + case ACOSH -> new MathACosHFunction(x); + case TANH -> new MathTanHFunction(x); + case ATANH -> new MathATanHFunction(x); + case CBRT -> new MathCbrtFunction(x); + case HYPOT -> new MathHypotFunction(x, y); + case FLOOR -> new MathFloorFunction(x); + }; } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java index 9f7cef456cf..fd9e8d6271a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -20,7 +20,7 @@ public class MathHypotFunction extends FunctionNode { } private MathHypotFunction(String label, Integer level, GroupingExpression x, GroupingExpression y) { - super("math.hypot", label, level, Arrays.asList(x, y)); + super("math.hypot", label, level, List.of(x, y)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java index 8f4a6444bfb..0f05a1909b1 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathLog10Function extends FunctionNode { } private MathLog10Function(String label, Integer level, GroupingExpression exp) { - super("math.log10", label, level, Arrays.asList(exp)); + super("math.log10", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java index 35fcbd3f1a6..500edd9f90b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathLog1pFunction extends FunctionNode { } private MathLog1pFunction(String label, Integer level, GroupingExpression exp) { - super("math.log1p", label, level, Arrays.asList(exp)); + super("math.log1p", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java index 5b06998b004..642c3230b3f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathLogFunction extends FunctionNode { } private MathLogFunction(String label, Integer level, GroupingExpression exp) { - super("math.log", label, level, Arrays.asList(exp)); + super("math.log", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java index 6f4972c9af0..7dd398f7008 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -20,7 +20,7 @@ public class MathPowFunction extends FunctionNode { } private MathPowFunction(String label, Integer level, GroupingExpression x, GroupingExpression y) { - super("math.pow", label, level, Arrays.asList(x, y)); + super("math.pow", label, level, List.of(x, y)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java index 8d21fb8e0df..64051cb96e8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathSinFunction extends FunctionNode { } private MathSinFunction(String label, Integer level, GroupingExpression exp) { - super("math.sin", label, level, Arrays.asList(exp)); + super("math.sin", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java index c9d30a0adec..21295278756 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathSinHFunction extends FunctionNode { } private MathSinHFunction(String label, Integer level, GroupingExpression exp) { - super("math.sinh", label, level, Arrays.asList(exp)); + super("math.sinh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java index 268ddaaaf2d..411a3fe30fa 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathSqrtFunction extends FunctionNode { } private MathSqrtFunction(String label, Integer level, GroupingExpression exp) { - super("math.sqrt", label, level, Arrays.asList(exp)); + super("math.sqrt", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java index 79267e0ae17..f2a18010258 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathTanFunction extends FunctionNode { } private MathTanFunction(String label, Integer level, GroupingExpression exp) { - super("math.tan", label, level, Arrays.asList(exp)); + super("math.tan", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java index e1a83c0b630..5833f0bf887 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class MathTanHFunction extends FunctionNode { } private MathTanHFunction(String label, Integer level, GroupingExpression exp) { - super("math.tanh", label, level, Arrays.asList(exp)); + super("math.tanh", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java index 720e071aebf..57604677b82 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents an md5-function in a {@link GroupingExpression}. It evaluates to a long that equals the md5 of @@ -22,7 +22,7 @@ public class Md5Function extends FunctionNode { } private Md5Function(String label, Integer level, GroupingExpression exp, LongValue numBits) { - super("md5", label, level, Arrays.asList(exp, numBits)); + super("md5", label, level, List.of(exp, numBits)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java index 73484de9e47..67ddd560ab9 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a minute-of-hour timestamp-function in a {@link GroupingExpression}. It evaluates to a long @@ -22,7 +22,7 @@ public class MinuteOfHourFunction extends FunctionNode { } private MinuteOfHourFunction(String label, Integer level, GroupingExpression exp) { - super("time.minuteofhour", label, level, Arrays.asList(exp)); + super("time.minuteofhour", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java index e8dace1e3bf..f9b4aef23f5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a month-of-year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that @@ -22,7 +22,7 @@ public class MonthOfYearFunction extends FunctionNode { } private MonthOfYearFunction(String label, Integer level, GroupingExpression exp) { - super("time.monthofyear", label, level, Arrays.asList(exp)); + super("time.monthofyear", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java index 9f8a8de0266..785d29301e4 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a negate-function in a {@link GroupingExpression}. It evaluates to a number that equals the @@ -22,7 +22,7 @@ public class NegFunction extends FunctionNode { } private NegFunction(String label, Integer level, GroupingExpression exp) { - super("neg", label, level, Arrays.asList(exp)); + super("neg", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java index 1b3f48052a4..c843959249d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** */ @@ -17,7 +17,7 @@ public class NormalizeSubjectFunction extends FunctionNode { } private NormalizeSubjectFunction(String label, Integer level, GroupingExpression exp) { - super("normalizesubject", label, level, Arrays.asList(exp)); + super("normalizesubject", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java index 7026e726675..a2e2bcd649a 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java @@ -1,8 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; /** * This class represents a now-function in a {@link GroupingExpression}. It evaluates to a long that equals the number @@ -21,7 +20,7 @@ public class NowFunction extends FunctionNode { } private NowFunction(String label, Integer level) { - super("now", label, level, Collections.emptyList()); + super("now", label, level, List.of()); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java index 483f2494ea9..819dd3601c4 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a reverse-function in a {@link GroupingExpression}. It evaluates to a list that equals the list @@ -22,7 +22,7 @@ public class ReverseFunction extends FunctionNode { } private ReverseFunction(String label, Integer level, GroupingExpression exp) { - super("reverse", label, level, Arrays.asList(exp)); + super("reverse", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java index a4ef16d50d3..fdcbeefe647 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a second-of-minute timestamp-function in a {@link GroupingExpression}. It evaluates to a long @@ -22,7 +22,7 @@ public class SecondOfMinuteFunction extends FunctionNode { } private SecondOfMinuteFunction(String label, Integer level, GroupingExpression exp) { - super("time.secondofminute", label, level, Arrays.asList(exp)); + super("time.secondofminute", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java index 418d1513973..33b910a1119 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a size-function in a {@link GroupingExpression}. It evaluates to a number that equals the @@ -22,7 +22,7 @@ public class SizeFunction extends FunctionNode { } private SizeFunction(String label, Integer level, GroupingExpression exp) { - super("size", label, level, Arrays.asList(exp)); + super("size", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java index 03b9479afda..0acee311b11 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a sort-function in a {@link GroupingExpression}. It evaluates to a list that equals the list @@ -22,7 +22,7 @@ public class SortFunction extends FunctionNode { } private SortFunction(String label, Integer level, GroupingExpression exp) { - super("sort", label, level, Arrays.asList(exp)); + super("sort", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java index 28b0f021b26..21679b01170 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a strcat-function in a {@link GroupingExpression}. It evaluates to a long that equals the @@ -22,7 +22,7 @@ public class StrLenFunction extends FunctionNode { } private StrLenFunction(String label, Integer level, GroupingExpression exp) { - super("strlen", label, level, Arrays.asList(exp)); + super("strlen", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java index 906e7ec18cb..6d05dd69a92 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a todouble-function in a {@link GroupingExpression}. It converts the result of the argument to @@ -22,7 +22,7 @@ public class ToDoubleFunction extends FunctionNode { } private ToDoubleFunction(String label, Integer level, GroupingExpression exp) { - super("todouble", label, level, Arrays.asList(exp)); + super("todouble", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java index 4c7db38442d..d92cd2814aa 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a tolong-function in a {@link GroupingExpression}. It converts the result of the argument to a @@ -22,7 +22,7 @@ public class ToLongFunction extends FunctionNode { } private ToLongFunction(String label, Integer level, GroupingExpression exp) { - super("tolong", label, level, Arrays.asList(exp)); + super("tolong", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java index 9c0a581fb87..99cf3400bdc 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a toraw-function in a {@link GroupingExpression}. It @@ -23,7 +23,7 @@ public class ToRawFunction extends FunctionNode { } private ToRawFunction(String label, Integer level, GroupingExpression exp) { - super("toraw", label, level, Arrays.asList(exp)); + super("toraw", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java index 0fb891bdce0..b58765c57f0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a tolong-function in a {@link GroupingExpression}. It converts the result of the argument to a @@ -22,7 +22,7 @@ public class ToStringFunction extends FunctionNode { } private ToStringFunction(String label, Integer level, GroupingExpression exp) { - super("tostring", label, level, Arrays.asList(exp)); + super("tostring", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java index c5e469f1379..98cb68fbb8c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -23,7 +22,7 @@ public class UcaFunction extends FunctionNode { * @param locale The locale to used for sorting. */ public UcaFunction(GroupingExpression exp, String locale) { - this(null, null, Arrays.asList(exp, new StringValue(locale))); + this(null, null, List.of(exp, new StringValue(locale))); } /** @@ -34,7 +33,7 @@ public class UcaFunction extends FunctionNode { * @param strength The strength level to use. */ public UcaFunction(GroupingExpression exp, String locale, String strength) { - this(null, null, Arrays.asList(exp, new StringValue(locale), new StringValue(strength))); + this(null, null, List.of(exp, new StringValue(locale), new StringValue(strength))); if ( ! validStrength(strength)) throw new IllegalArgumentException("Not a valid UCA strength: " + strength); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java index 2e0bf25b894..f03bf681e90 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -21,7 +20,7 @@ public class XorBitFunction extends FunctionNode { * @param numBits The number of bits of the expression value to xor. */ public XorBitFunction(GroupingExpression exp, int numBits) { - this(null, null, Arrays.asList(exp, new LongValue(numBits))); + this(null, null, List.of(exp, new LongValue(numBits))); } private XorBitFunction(String label, Integer level, List<GroupingExpression> exp) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java index 150b54855f3..528a6253ab0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * This class represents a year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that equals @@ -22,7 +22,7 @@ public class YearFunction extends FunctionNode { } private YearFunction(String label, Integer level, GroupingExpression exp) { - super("time.year", label, level, Arrays.asList(exp)); + super("time.year", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java index 29216e20825..e954e2c24fc 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -19,7 +19,7 @@ public class ZCurveXFunction extends FunctionNode { } private ZCurveXFunction(String label, Integer level, GroupingExpression exp) { - super("zcurve.x", label, level, Arrays.asList(exp)); + super("zcurve.x", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java index 4803f145487..edbc9417a6b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; -import java.util.Arrays; +import java.util.List; /** * @author baldersheim @@ -18,7 +18,7 @@ public class ZCurveYFunction extends FunctionNode { } private ZCurveYFunction(String label, Integer level, GroupingExpression exp) { - super("zcurve.y", label, level, Arrays.asList(exp)); + super("zcurve.y", label, level, List.of(exp)); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index be4f2f786e5..fba9064298c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -21,7 +21,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -357,7 +356,7 @@ public class GroupingExecutor extends Searcher { public static List<Grouping> getGroupingList(Query query) { Object obj = query.properties().get(PROP_GROUPINGLIST); if (!(obj instanceof List)) { - return Collections.emptyList(); + return List.of(); } return (List<Grouping>)obj; } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index c1eff1eec67..c2a5ad0222d 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -3,7 +3,6 @@ package com.yahoo.search.handler; import ai.vespa.metrics.ContainerMetrics; import ai.vespa.cloud.ZoneInfo; -import ai.vespa.metrics.ContainerMetrics; import com.yahoo.collections.Tuple2; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Vtag; @@ -117,7 +116,7 @@ public class SearchHandler extends LoggingRequestHandler { ZoneInfo zoneInfo) { this(metric, threadpool.executor(), queryProfileRegistry, embedders, executionFactory, config.numQueriesToTraceOnDebugAfterConstruction(), - config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey()), + config.hostResponseHeaderKey().isEmpty() ? Optional.empty() : Optional.of(config.hostResponseHeaderKey()), zoneInfo); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java index e07b290f66e..22bc97b9ca1 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java @@ -22,8 +22,6 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -99,7 +97,7 @@ public class PageTemplateSearcher extends Searcher { * @param resolvers the resolvers to use, in addition to the default resolvers */ public PageTemplateSearcher(PageTemplateRegistry templateRegistry, Resolver... resolvers) { - this(templateRegistry, Arrays.asList(resolvers)); + this(templateRegistry, List.of(resolvers)); } private PageTemplateSearcher(PageTemplateRegistry templateRegistry, List<Resolver> resolvers) { @@ -139,13 +137,13 @@ public class PageTemplateSearcher extends Searcher { if (pageIds == null) { String pageIdString = query.properties().getString(pageIdName,"").trim(); if (!pageIdString.isEmpty()) - pageIds = Arrays.asList(pageIdString.split(" ")); + pageIds = List.of(pageIdString.split(" ")); } // If none set, just return the default or null if none if (pageIds == null) { PageElement defaultPage=templateRegistry.getComponent("default"); - return (defaultPage == null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage)); + return (defaultPage == null ? List.of() : List.of(defaultPage)); } // Resolve the id list to page templates diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java index f59124e293e..3ee195bdf55 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java @@ -72,7 +72,7 @@ public class PageTemplateXMLReader { File file = new File(fileName); pageReader = new NamedReader(fileName,new FileReader(file)); String firstName = file.getName().substring(0, file.getName().length() - 4); - return read(Collections.singletonList(pageReader), true).getComponent(firstName); + return read(List.of(pageReader), true).getComponent(firstName); } catch (IOException e) { throw new IllegalArgumentException("Could not read the page template '" + fileName + "'", e); @@ -85,7 +85,7 @@ public class PageTemplateXMLReader { private List<File> sortFiles(File dir) { ArrayList<File> files = new ArrayList<>(); - files.addAll(Arrays.asList(dir.listFiles())); + files.addAll(List.of(dir.listFiles())); Collections.sort(files); return files; } @@ -296,7 +296,7 @@ public class PageTemplateXMLReader { if ("item".equals(value.getNodeName())) map.values().add(readPageElements(value)); else - map.values().add(Collections.singletonList(readPageElement(value))); + map.values().add(List.of(readPageElement(value))); } return map; } @@ -311,7 +311,7 @@ public class PageTemplateXMLReader { else if (alternative.getNodeName().equals("include")) // Implicit include choice.alternatives().add(readInclude(alternative)); else // Other implicit - choice.alternatives().add(Collections.singletonList(readPageElement(alternative))); + choice.alternatives().add(List.of(readPageElement(alternative))); } return choice; } @@ -323,7 +323,7 @@ public class PageTemplateXMLReader { if ("alternative".equals(alternative.getNodeName())) // Explicit alternative container source.renderer().alternatives().addAll(readRenderers(XML.children(alternative))); else // Implicit alternative - yes implicit and explicit may be combined - source.renderer().alternatives().addAll(readRenderers(Collections.singletonList(alternative))); + source.renderer().alternatives().addAll(readRenderers(List.of(alternative))); } } } @@ -331,7 +331,7 @@ public class PageTemplateXMLReader { private Map<String,String> readParameters(Element containingElement) { List<Element> parameterElements=XML.getChildren(containingElement,"parameter"); - if (parameterElements.size()==0) return Collections.emptyMap(); // Shortcut + if (parameterElements.size()==0) return Map.of(); // Shortcut Map<String,String> parameters=new LinkedHashMap<>(); for (Element parameter : parameterElements) { diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 32e28dc3ff7..6735a6bd050 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -10,7 +10,6 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.yql.VespaGroupingStep; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -53,7 +52,7 @@ public class Select implements Cloneable { } public Select(String where, String grouping, Query query) { - this(where, grouping, null, query, Collections.emptyList()); + this(where, grouping, null, query, List.of()); } private Select(String where, String grouping, String groupingExpressionString, Query query, List<GroupingRequest> groupingRequests) { diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 93df7fdfb18..c90612425fa 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -55,7 +55,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -147,7 +146,7 @@ public class SelectParser implements Parser { private final Normalizer normalizer; private IndexFacts.Session indexFactsSession; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = List.of(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -1151,8 +1150,9 @@ public class SelectParser implements Parser { Integer maxEditDistance = getIntegerAnnotation(MAX_EDIT_DISTANCE, annotations, FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE); Integer prefixLength = getIntegerAnnotation(PREFIX_LENGTH, annotations, FuzzyItem.DEFAULT_PREFIX_LENGTH); + boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE); - FuzzyItem fuzzy = new FuzzyItem(field, true, wordData, maxEditDistance, prefixLength); + FuzzyItem fuzzy = new FuzzyItem(field, true, wordData, maxEditDistance, prefixLength, prefixMatch); return leafStyleSettings(getAnnotations(value), fuzzy); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java index 53f8ccbe6de..3587175b671 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java @@ -66,7 +66,7 @@ public class CopyOnWriteContent extends FreezableClass implements Cloneable { public void freeze() { // Freeze this if (unmodifiableMap==null) - unmodifiableMap= map!=null ? Collections.unmodifiableMap(map) : Collections.<String, Object>emptyMap(); + unmodifiableMap= map!=null ? Collections.unmodifiableMap(map) : Map.of(); map=null; // just to keep the states simpler // Freeze content @@ -119,7 +119,7 @@ public class CopyOnWriteContent extends FreezableClass implements Cloneable { //------- Content access ------------------------------------------------------- public Map<String,Object> unmodifiableMap() { - if (isEmpty()) return Collections.emptyMap(); + if (isEmpty()) return Map.of(); if (map==null) // in COPYONWRITE or FROZEN state return unmodifiableMap; // In WRITABLE state: Create unmodifiable wrapper if necessary and return it diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index 1c694417475..d638f57422c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -121,7 +121,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable */ public List<QueryProfile> inherited() { if (isFrozen()) return inherited; // Frozen profiles always have an unmodifiable, non-null list - if (inherited == null) return Collections.emptyList(); + if (inherited == null) return List.of(); return Collections.unmodifiableList(inherited); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 8778dcc7348..261e42a972d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -18,7 +18,6 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.tensor.Tensor; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -127,7 +126,7 @@ public class QueryProfileProperties extends Properties { name = unalias(name, context); if (context == null) - context = Collections.emptyMap(); + context = Map.of(); if ( ! profile.isOverridable(name, context)) return; @@ -237,7 +236,7 @@ public class QueryProfileProperties extends Properties { context = contextWithZoneInfo(context); path = unalias(path, context); - if (context == null) context = Collections.emptyMap(); + if (context == null) context = Map.of(); Map<String, Object> properties = new HashMap<>(); for (var entry : profile.listValues(path, context, substitution).entrySet()) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java index 0ed6b590227..c9dbfe9d89a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java @@ -5,7 +5,6 @@ import com.yahoo.component.provider.Freezable; import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -187,7 +186,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { * @param dimensionBinding the dimension bindings to use in this */ public Object get(String name, QueryProfileType type, boolean allowQueryProfileResult, DimensionBinding dimensionBinding) { - SingleValueQueryProfileVisitor visitor = new SingleValueQueryProfileVisitor(Collections.singletonList(name),allowQueryProfileResult); + SingleValueQueryProfileVisitor visitor = new SingleValueQueryProfileVisitor(List.of(name),allowQueryProfileResult); visitor.enter(""); accept(true, type, visitor, dimensionBinding); visitor.leave(""); @@ -370,7 +369,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { /** Returns the field values (values for various dimensions) for this field as a read-only list (never null) */ public List<FieldValue> asList() { - if (resolutionList == null) return Collections.emptyList(); + if (resolutionList == null) return List.of(); return resolutionList; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 4b3c3ede1e9..562d59846d1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -9,7 +9,6 @@ import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.SubstituteString; import com.yahoo.search.query.profile.types.QueryProfileType; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -102,7 +101,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.emptyMap()); } + public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Map.of()); } public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index b6fa58628a1..0823989a104 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -2,18 +2,15 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.DimensionBinding; import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * Contains the values a given key in a DimensionalMap may take for different dimensional contexts. @@ -44,7 +41,7 @@ public class DimensionalValue<VALUE> { /** Returns the value matching this context, or null if none */ public VALUE get(Map<String, String> context) { if (context == null) - context = Collections.emptyMap(); + context = Map.of(); for (BindingSpec spec : bindingSpecs) { if ( ! spec.matches(context)) continue; @@ -156,8 +153,7 @@ public class DimensionalValue<VALUE> { variants.add(binding); // We're combining values for efficiency, so remove incorrect provenance info - if (value instanceof ValueWithSource) { - ValueWithSource v1 = (ValueWithSource)value; + if (value instanceof ValueWithSource v1) { ValueWithSource v2 = (ValueWithSource)newValue; if (v1.source() != null && ! v1.source().equals(v2.source())) @@ -220,14 +216,12 @@ public class DimensionalValue<VALUE> { private VALUE substituteIfRelative(VALUE value, Binding variant, Map<CompoundName, DimensionalValue.Builder<VALUE>> entries) { - if (value instanceof ValueWithSource && ((ValueWithSource)value).value() instanceof SubstituteString) { - ValueWithSource valueWithSource = (ValueWithSource)value; + if (value instanceof ValueWithSource valueWithSource && ((ValueWithSource)value).value() instanceof SubstituteString) { SubstituteString substitute = (SubstituteString)valueWithSource.value(); if (substitute.hasRelative()) { List<SubstituteString.Component> resolvedComponents = new ArrayList<>(substitute.components().size()); for (SubstituteString.Component component : substitute.components()) { - if (component instanceof SubstituteString.RelativePropertyComponent) { - SubstituteString.RelativePropertyComponent relativeComponent = (SubstituteString.RelativePropertyComponent)component; + if (component instanceof SubstituteString.RelativePropertyComponent relativeComponent) { var substituteValues = lookupByLocalName(relativeComponent.fieldName(), entries); if (substituteValues == null) throw new IllegalArgumentException("Could not resolve local substitution '" + @@ -274,8 +268,8 @@ public class DimensionalValue<VALUE> { /** Returns whether this context contains all the keys of this */ public boolean matches(Map<String, String> context) { - for (int i = 0; i < dimensions.length; i++) - if ( ! context.containsKey(dimensions[i])) return false; + for (String dimension : dimensions) + if (!context.containsKey(dimension)) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index f2eb76f2367..abb0129eed0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -18,7 +18,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -68,7 +67,7 @@ public class QueryProfileXMLReader { private List<File> sortFiles(File dir) { ArrayList<File> files = new ArrayList<>(); - files.addAll(Arrays.asList(dir.listFiles())); + files.addAll(List.of(dir.listFiles())); Collections.sort(files); return files; } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java index 080f938d61a..e5319b50b4f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.textserialize.item; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -19,7 +18,7 @@ public class ItemArguments { properties = (Map<?, ?>) ListUtil.first(arguments); children = ListUtil.rest(arguments); } else { - properties = Collections.emptyMap(); + properties = Map.of(); children = arguments; } } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java index c8c9f7241d2..7971bc0a30e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java @@ -7,7 +7,6 @@ import com.yahoo.prelude.query.TaggableItem; import com.yahoo.search.query.textserialize.serializer.DispatchForm; import com.yahoo.search.query.textserialize.serializer.ItemIdMapper; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -116,7 +115,7 @@ public class ItemInitializer { Item connectedItem = taggableItem.getConnectedItem(); if (connectedItem != null) { form.setProperty("connectivity", - Arrays.asList(itemIdMapper.getId(connectedItem), taggableItem.getConnectivity())); + List.of(itemIdMapper.getId(connectedItem), taggableItem.getConnectivity())); } if (taggableItem.hasExplicitSignificance()) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java index 83ae349f5a0..88a1e6c1485 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java @@ -64,7 +64,17 @@ public class EventRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void data(Data data) throws IOException { - if (data instanceof EventStream.Event event) { + if (data instanceof EventStream.ErrorEvent error) { + generator.writeRaw("event: error\n"); + generator.writeRaw("data: "); + generator.writeStartObject(); + generator.writeStringField("source", error.source()); + generator.writeNumberField("error", error.code()); + generator.writeStringField("message", error.message()); + generator.writeEndObject(); + generator.writeRaw("\n\n"); + generator.flush(); + } else if (data instanceof EventStream.Event event) { if (RENDER_EVENT_HEADER) { generator.writeRaw("event: " + event.type() + "\n"); } @@ -75,19 +85,6 @@ public class EventRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeRaw("\n\n"); generator.flush(); } - else if (data instanceof ErrorHit) { - for (ErrorMessage error : ((ErrorHit) data).errors()) { - generator.writeRaw("event: error\n"); - generator.writeRaw("data: "); - generator.writeStartObject(); - generator.writeStringField("source", error.getSource()); - generator.writeNumberField("error", error.getCode()); - generator.writeStringField("message", error.getMessage()); - generator.writeEndObject(); - generator.writeRaw("\n\n"); - generator.flush(); - } - } // Todo: support other types of data such as search results (hits), timing and trace } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index d62860afcda..74e79475ce6 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -10,7 +10,7 @@ import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.concurrent.Executor; /** @@ -34,7 +34,7 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi * Use MoreExecutors.directExecutor(). */ public RendererRegistry(Executor executor) { - this(Collections.emptyList(), executor); + this(List.of(), executor); } /** @@ -111,7 +111,7 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi private String rendererNames() { StringBuilder r = new StringBuilder(); for (Renderer<Result> c : allComponents()) { - if (r.length() > 0) + if (!r.isEmpty()) r.append(", "); r.append(c.getId().stringValue()); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java index 6f376042f2d..7bd3ca1174f 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java @@ -15,7 +15,6 @@ import java.io.IOException; import java.io.Writer; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; @@ -156,7 +155,7 @@ abstract public class SectionedRenderer<WRITER> extends Renderer { } private void renderResultContent(WRITER writer, Result result) throws IOException { - if (result.hits().getError() != null || result.hits().getQuery().errors().size() > 0) { + if (result.hits().getError() != null || !result.hits().getQuery().errors().isEmpty()) { error(writer, asUnmodifiableSearchErrorList(result.hits().getQuery().errors(), result.hits().getError())); } @@ -172,14 +171,14 @@ abstract public class SectionedRenderer<WRITER> extends Renderer { } private Collection<ErrorMessage> asUnmodifiableSearchErrorList(List<com.yahoo.processing.request.ErrorMessage> queryErrors,ErrorMessage resultError) { - if (queryErrors.size() == 0) - return Collections.singletonList(resultError); + if (queryErrors.isEmpty()) + return List.of(resultError); List<ErrorMessage> searchErrors = new ArrayList<>(queryErrors.size() + (resultError != null ? 1 :0) ); - for (int i=0; i<queryErrors.size(); i++) - searchErrors.add(ErrorMessage.from(queryErrors.get(i))); + for (com.yahoo.processing.request.ErrorMessage queryError : queryErrors) + searchErrors.add(ErrorMessage.from(queryError)); if (resultError != null) searchErrors.add(resultError); - return Collections.unmodifiableCollection(searchErrors); + return List.copyOf(searchErrors); } private void renderHitGroup(WRITER writer, HitGroup hitGroup) throws IOException { diff --git a/container-search/src/main/java/com/yahoo/search/result/EventStream.java b/container-search/src/main/java/com/yahoo/search/result/EventStream.java index b393a91e6d0..8e6f7977d55 100644 --- a/container-search/src/main/java/com/yahoo/search/result/EventStream.java +++ b/container-search/src/main/java/com/yahoo/search/result/EventStream.java @@ -41,7 +41,7 @@ public class EventStream extends Hit implements DataList<Data> { } public void error(String source, ErrorMessage message) { - incoming().add(new DefaultErrorHit(source, message)); + incoming().add(new ErrorEvent(eventCount.incrementAndGet(), source, message)); } public void markComplete() { @@ -117,4 +117,38 @@ public class EventStream extends Hit implements DataList<Data> { } + public static class ErrorEvent extends Event { + + private final String source; + private final ErrorMessage message; + + public ErrorEvent(int eventNumber, String source, ErrorMessage message) { + super(eventNumber, message.getMessage(), "error"); + this.source = source; + this.message = message; + } + + public String source() { + return source; + } + + public int code() { + return message.getCode(); + } + + public String message() { + return message.getMessage(); + } + + @Override + public Hit asHit() { + Hit hit = super.asHit(); + hit.setField("source", source); + hit.setField("code", message.getCode()); + return hit; + } + + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 0f26ecdcce7..55a4c6c11f2 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -14,7 +14,6 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -150,7 +149,7 @@ public class FeatureData implements Inspectable, JsonProducer { /** Returns the names of the features available in this */ public Set<String> featureNames() { - if (this == empty) return Collections.emptySet(); + if (this == empty) return Set.of(); if (featureNames != null) return featureNames; featureNames = new HashSet<>(); diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index d81880cc548..4d3b8bb0884 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -333,6 +333,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi */ public void setFillable() { if (filled == null) { + //TODO Should have used Set.of(), but it checks that contains is not called with null filled = Collections.emptySet(); unmodifiableFilled = filled; } @@ -347,6 +348,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi */ public void setFilled(String summaryClass) { if (filled == null || filled.isEmpty()) { + //TODO Should have used Set.of(), but it checks that contains is not called with null filled = Collections.singleton(summaryClass); unmodifiableFilled = filled; } else if (filled.size() == 1) { @@ -483,7 +485,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String, Object> getUnmodifiableFieldMap() { if (unmodifiableFieldMap == null) { if (fields == null) { - return Collections.emptyMap(); + return Map.of(); } else { unmodifiableFieldMap = Collections.unmodifiableMap(fields); } @@ -574,12 +576,12 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Attach some data to this hit for this searcher */ public void setSearcherSpecificMetaData(Searcher searcher, Object data) { if (searcherSpecificMetaData == null) { - searcherSpecificMetaData = Collections.singletonMap(searcher, data); + if (data != null) + searcherSpecificMetaData = Map.of(searcher, data); } else { if (searcherSpecificMetaData.size() == 1) { - Object tmp = searcherSpecificMetaData.get(searcher); - if (tmp != null) { - searcherSpecificMetaData = Collections.singletonMap(searcher, data); + if (searcherSpecificMetaData.containsKey(searcher)) { + searcherSpecificMetaData = (data != null) ? Map.of(searcher, data) : null; } else { searcherSpecificMetaData = new TreeMap<>(searcherSpecificMetaData); searcherSpecificMetaData.put(searcher, data); diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 51c0caf38a9..be31b91a304 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -718,6 +718,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< if (hitFilled.isEmpty()) { filled = null; } else if (hitFilled.size() == 1) { + //TODO Avoid needing set that allows null .... filled = Collections.singleton(hitFilled.iterator().next()); } else { filled = new HashSet<>(hitFilled); @@ -799,6 +800,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< analyzeHit(hit); Set<String> hitFilled = hit.getFilled(); if (hitFilled != null) { + //TODO Avoid needing set that allows null .... filled = (hitFilled.size() == 1) ? Collections.singleton(hitFilled.iterator().next()) : hitFilled.isEmpty() ? null : new HashSet<>(hitFilled); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java index 27d7430a0d9..9698735fb52 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java @@ -6,7 +6,6 @@ import com.yahoo.component.chain.Chain; import com.yahoo.component.chain.Phase; import com.yahoo.search.Searcher; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -33,7 +32,7 @@ public class SearchChain extends Chain<Searcher> { } public SearchChain(ComponentId id, Searcher... searchers) { - this(id, Arrays.asList(searchers)); + this(id, List.of(searchers)); } public SearchChain(ComponentId id, Collection<Searcher> searchers) { diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index f21ea506fde..69a1f8ec6cb 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -11,7 +11,6 @@ import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -61,7 +60,7 @@ public class VespaSearchers { private static FederationSearcherModel federationSearcherModel() { return new FederationSearcherModel(new ComponentSpecification("federation"), Dependencies.emptyDependencies(), - Collections.emptyList(), + List.of(), true); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 634163bf0c2..a354006aa9b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -551,24 +551,31 @@ public class VespaSerializer { static String fuzzyAnnotations(FuzzyItem fuzzyItem) { boolean isMaxEditDistanceSet = fuzzyItem.getMaxEditDistance() != FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE; boolean isPrefixLengthSet = fuzzyItem.getPrefixLength() != FuzzyItem.DEFAULT_PREFIX_LENGTH; - boolean anyAnnotationSet = isMaxEditDistanceSet || isPrefixLengthSet; + boolean isPrefixMatch = fuzzyItem.isPrefixMatch(); + boolean anyAnnotationSet = isMaxEditDistanceSet || isPrefixLengthSet || isPrefixMatch; - StringBuilder builder = new StringBuilder(); - if (anyAnnotationSet) { - builder.append("{"); + if (!anyAnnotationSet) { + return ""; } + + StringBuilder builder = new StringBuilder(); + builder.append("{"); if (isMaxEditDistanceSet) { builder.append(MAX_EDIT_DISTANCE + ":").append(fuzzyItem.getMaxEditDistance()); - } - if (isMaxEditDistanceSet && isPrefixLengthSet) { - builder.append(","); + if (isPrefixLengthSet || isPrefixMatch) { + builder.append(","); + } } if (isPrefixLengthSet) { builder.append(PREFIX_LENGTH + ":").append(fuzzyItem.getPrefixLength()); + if (isPrefixMatch) { + builder.append(","); + } } - if (anyAnnotationSet) { - builder.append("}"); + if (isPrefixMatch) { + builder.append(PREFIX).append(':').append(fuzzyItem.isPrefixMatch()); } + builder.append("}"); return builder.toString(); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 7ae02c18e7a..fb4ec5ba872 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -913,7 +913,7 @@ public class YqlParser implements Parser { GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); List<Object> continuations = getAnnotation(groupingAst, "continuations", List.class, - Collections.emptyList(), "grouping continuations"); + List.of(), "grouping continuations"); for (Object continuation : continuations) { groupingStep.continuations().add(Continuation.fromString(dereference(continuation))); @@ -1385,7 +1385,14 @@ public class YqlParser implements Parser { FuzzyItem.DEFAULT_PREFIX_LENGTH, PREFIX_LENGTH_DESCRIPTION); - FuzzyItem fuzzy = new FuzzyItem(field, true, wordData, maxEditDistance, prefixLength); + boolean prefixMatch = getAnnotation( + ast, + PREFIX, + Boolean.class, + Boolean.FALSE, + "setting for whether to use prefix match of input data"); + + FuzzyItem fuzzy = new FuzzyItem(field, true, wordData, maxEditDistance, prefixLength, prefixMatch); return leafStyleSettings(ast, fuzzy); } @@ -1614,7 +1621,7 @@ public class YqlParser implements Parser { { Item leaf = (Item) out; Map<?, ?> itemAnnotations = getAnnotation(ast, ANNOTATIONS, - Map.class, Collections.emptyMap(), "item annotation map"); + Map.class, Map.of(), "item annotation map"); for (Map.Entry<?, ?> entry : itemAnnotations.entrySet()) { Preconditions.checkArgument(entry.getKey() instanceof String, "Expected String annotation key, got %s.", entry.getKey().getClass()); diff --git a/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java b/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java index 561d2bf34d0..c957432a957 100644 --- a/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java +++ b/container-search/src/main/java/com/yahoo/text/interpretation/Annotations.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text.interpretation; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -28,7 +27,7 @@ public class Annotations { public Map<String,Object> getMap() { if (annotations == null) { - return Collections.emptyMap(); + return Map.of(); } else { return annotations; } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java index 112c3669133..c0ded6a4c39 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java @@ -8,7 +8,6 @@ import com.yahoo.search.Query; import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -68,10 +67,10 @@ class QueryEncoder { return cacheSettingMap; } if (query.getGroupingSessionCache()) - return Collections.singletonMap("grouping", true); + return Map.of("grouping", true); if (query.getRanking().getQueryCache()) - return Collections.singletonMap("query", true); - return Collections.emptyMap(); + return Map.of("query", true); + return Map.of(); } private static Map<String, String> createModelMap(Query query) { diff --git a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java index 1efcf1c736a..3baa9715c34 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java @@ -3,14 +3,11 @@ package ai.vespa.search.llm; import ai.vespa.llm.InferenceParameters; import ai.vespa.llm.LanguageModel; -import ai.vespa.llm.clients.ConfigurableLanguageModelTest; -import ai.vespa.llm.clients.LlmClientConfig; -import ai.vespa.llm.clients.MockLLMClient; +import ai.vespa.llm.completion.Completion; import ai.vespa.llm.completion.Prompt; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.jdisc.SecretStoreProvider; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -20,10 +17,14 @@ import org.junit.jupiter.api.Test; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -36,10 +37,10 @@ public class LLMSearcherTest { @Test public void testLLMSelection() { - var llm1 = createLLMClient("mock1"); - var llm2 = createLLMClient("mock2"); + var client1 = createLLMClient("mock1"); + var client2 = createLLMClient("mock2"); var config = new LlmSearcherConfig.Builder().stream(false).providerId("mock2").build(); - var searcher = createLLMSearcher(config, Map.of("mock1", llm1, "mock2", llm2)); + var searcher = createLLMSearcher(config, Map.of("mock1", client1, "mock2", client2)); var result = runMockSearch(searcher, Map.of("prompt", "what is your id?")); assertEquals(1, result.getHitCount()); assertEquals("My id is mock2", getCompletion(result)); @@ -47,14 +48,16 @@ public class LLMSearcherTest { @Test public void testGeneration() { - var searcher = createLLMSearcher(Map.of("mock", createLLMClient())); + var client = createLLMClient(); + var searcher = createLLMSearcher(client); var params = Map.of("prompt", "why are ducks better than cats"); assertEquals("Ducks have adorable waddling walks.", getCompletion(runMockSearch(searcher, params))); } @Test public void testPrompting() { - var searcher = createLLMSearcher(Map.of("mock", createLLMClient())); + var client = createLLMClient(); + var searcher = createLLMSearcher(client); // Prompt with prefix assertEquals("Ducks have adorable waddling walks.", @@ -71,7 +74,8 @@ public class LLMSearcherTest { @Test public void testPromptEvent() { - var searcher = createLLMSearcher(Map.of("mock", createLLMClient())); + var client = createLLMClient(); + var searcher = createLLMSearcher(client); var params = Map.of( "prompt", "why are ducks better than cats", "traceLevel", "1"); @@ -90,7 +94,8 @@ public class LLMSearcherTest { @Test public void testParameters() { - var searcher = createLLMSearcher(Map.of("mock", createLLMClient())); + var client = createLLMClient(); + var searcher = createLLMSearcher(client); var params = Map.of( "llm.prompt", "why are ducks better than cats", "llm.temperature", "1.0", @@ -107,16 +112,18 @@ public class LLMSearcherTest { "foo.maxTokens", "5" ); var config = new LlmSearcherConfig.Builder().stream(false).propertyPrefix(prefix).providerId("mock").build(); - var searcher = createLLMSearcher(config, Map.of("mock", createLLMClient())); + var client = createLLMClient(); + var searcher = createLLMSearcher(config, client); assertEquals("I have no opinion on", getCompletion(runMockSearch(searcher, params))); } @Test public void testApiKeyFromHeader() { var properties = Map.of("prompt", "why are ducks better than cats"); - var searcher = createLLMSearcher(Map.of("mock", createLLMClientWithoutSecretStore())); - assertThrows(IllegalArgumentException.class, () -> runMockSearch(searcher, properties, "invalid_key", "llm")); - assertDoesNotThrow(() -> runMockSearch(searcher, properties, MockLLMClient.ACCEPTED_API_KEY, "llm")); + var client = createLLMClient(createApiKeyGenerator("a_valid_key")); + var searcher = createLLMSearcher(client); + assertThrows(IllegalArgumentException.class, () -> runMockSearch(searcher, properties, "invalid_key")); + assertDoesNotThrow(() -> runMockSearch(searcher, properties, "a_valid_key")); } @Test @@ -129,7 +136,8 @@ public class LLMSearcherTest { "llm.stream", "true", // ... but inference parameters says do it anyway "llm.prompt", "why are ducks better than cats?" ); - var searcher = createLLMSearcher(config, Map.of("mock", createLLMClient(executor))); + var client = createLLMClient(executor); + var searcher = createLLMSearcher(config, client); Result result = runMockSearch(searcher, params); assertEquals(1, result.getHitCount()); @@ -162,6 +170,10 @@ public class LLMSearcherTest { return runMockSearch(searcher, parameters, null, ""); } + static Result runMockSearch(Searcher searcher, Map<String, String> parameters, String apiKey) { + return runMockSearch(searcher, parameters, apiKey, "llm"); + } + static Result runMockSearch(Searcher searcher, Map<String, String> parameters, String apiKey, String prefix) { Chain<Searcher> chain = new Chain<>(searcher); Execution execution = new Execution(chain, Execution.Context.createContextStub()); @@ -191,43 +203,59 @@ public class LLMSearcherTest { } private static BiFunction<Prompt, InferenceParameters, String> createGenerator() { - return ConfigurableLanguageModelTest.createGenerator(); + return (prompt, options) -> { + String answer = "I have no opinion on the matter"; + if (prompt.asString().contains("ducks")) { + answer = "Ducks have adorable waddling walks."; + var temperature = options.getDouble("temperature"); + if (temperature.isPresent() && temperature.get() > 0.5) { + answer = "Random text about ducks vs cats that makes no sense whatsoever."; + } + } + var maxTokens = options.getInt("maxTokens"); + if (maxTokens.isPresent()) { + return Arrays.stream(answer.split(" ")).limit(maxTokens.get()).collect(Collectors.joining(" ")); + } + return answer; + }; } - static MockLLMClient createLLMClient() { - var config = new LlmClientConfig.Builder().apiKeySecretName("api-key").build(); - var secretStore = ConfigurableLanguageModelTest.createSecretStore(Map.of("api-key", MockLLMClient.ACCEPTED_API_KEY)); - var generator = createGenerator(); - return new MockLLMClient(config, secretStore, generator, null); + private static BiFunction<Prompt, InferenceParameters, String> createApiKeyGenerator(String validApiKey) { + return (prompt, options) -> { + if (options.getApiKey().isEmpty() || ! options.getApiKey().get().equals(validApiKey)) { + throw new IllegalArgumentException("Invalid API key"); + } + return "Ok"; + }; + } + + static MockLLM createLLMClient() { + return new MockLLM(createGenerator(), null); } - static MockLLMClient createLLMClient(String id) { - var config = new LlmClientConfig.Builder().apiKeySecretName("api-key").build(); - var secretStore = ConfigurableLanguageModelTest.createSecretStore(Map.of("api-key", MockLLMClient.ACCEPTED_API_KEY)); - var generator = createIdGenerator(id); - return new MockLLMClient(config, secretStore, generator, null); + static MockLLM createLLMClient(String id) { + return new MockLLM(createIdGenerator(id), null); } - static MockLLMClient createLLMClient(ExecutorService executor) { - var config = new LlmClientConfig.Builder().apiKeySecretName("api-key").build(); - var secretStore = ConfigurableLanguageModelTest.createSecretStore(Map.of("api-key", MockLLMClient.ACCEPTED_API_KEY)); - var generator = createGenerator(); - return new MockLLMClient(config, secretStore, generator, executor); + static MockLLM createLLMClient(BiFunction<Prompt, InferenceParameters, String> generator) { + return new MockLLM(generator, null); } - static MockLLMClient createLLMClientWithoutSecretStore() { - var config = new LlmClientConfig.Builder().apiKeySecretName("api-key").build(); - var secretStore = new SecretStoreProvider(); - var generator = createGenerator(); - return new MockLLMClient(config, secretStore.get(), generator, null); + static MockLLM createLLMClient(ExecutorService executor) { + return new MockLLM(createGenerator(), executor); + } + + private static Searcher createLLMSearcher(LanguageModel llm) { + return createLLMSearcher(Map.of("mock", llm)); } private static Searcher createLLMSearcher(Map<String, LanguageModel> llms) { var config = new LlmSearcherConfig.Builder().stream(false).build(); - ComponentRegistry<LanguageModel> models = new ComponentRegistry<>(); - llms.forEach((key, value) -> models.register(ComponentId.fromString(key), value)); - models.freeze(); - return new LLMSearcher(config, models); + return createLLMSearcher(config, llms); + } + + private static Searcher createLLMSearcher(LlmSearcherConfig config, LanguageModel llm) { + return createLLMSearcher(config, Map.of("mock", llm)); } private static Searcher createLLMSearcher(LlmSearcherConfig config, Map<String, LanguageModel> llms) { @@ -237,4 +265,44 @@ public class LLMSearcherTest { return new LLMSearcher(config, models); } + private static class MockLLM implements LanguageModel { + + private final ExecutorService executor; + private final BiFunction<Prompt, InferenceParameters, String> generator; + + public MockLLM(BiFunction<Prompt, InferenceParameters, String> generator, ExecutorService executor) { + this.executor = executor; + this.generator = generator; + } + + @Override + public List<Completion> complete(Prompt prompt, InferenceParameters params) { + return List.of(Completion.from(this.generator.apply(prompt, params))); + } + + @Override + public CompletableFuture<Completion.FinishReason> completeAsync(Prompt prompt, + InferenceParameters params, + Consumer<Completion> consumer) { + var completionFuture = new CompletableFuture<Completion.FinishReason>(); + var completions = this.generator.apply(prompt, params).split(" "); // Simple tokenization + + long sleep = 1; + executor.submit(() -> { + try { + for (int i = 0; i < completions.length; ++i) { + String completion = (i > 0 ? " " : "") + completions[i]; + consumer.accept(Completion.from(completion, Completion.FinishReason.none)); + Thread.sleep(sleep); + } + completionFuture.complete(Completion.FinishReason.stop); + } catch (InterruptedException e) { + // Do nothing + } + }); + return completionFuture; + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java b/container-search/src/test/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java index 150a2a5d78a..ebe4ccdd2ba 100644 --- a/container-search/src/test/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java +++ b/container-search/src/test/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java @@ -8,8 +8,6 @@ import org.osgi.framework.Bundle; import java.util.Collection; import java.util.List; -import static java.util.Collections.emptyList; - /** * @author gjoranv */ @@ -22,7 +20,7 @@ public class MockOsgiWrapper implements OsgiWrapper { @Override public List<Bundle> getCurrentBundles() { - return emptyList(); + return List.of(); } @Override @@ -32,7 +30,7 @@ public class MockOsgiWrapper implements OsgiWrapper { @Override public List<Bundle> install(String absolutePath) { - return emptyList(); + return List.of(); } @Override diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java index 58427bee30a..3cefeeabdcf 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/IndexedBackendTestCase.java @@ -20,7 +20,6 @@ import com.yahoo.search.schema.Schema; import com.yahoo.search.schema.SchemaInfo; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -98,7 +97,7 @@ public class IndexedBackendTestCase { var backend = new IndexedBackend(new ClusterParams(CLUSTER_PARAMS.getSearcherName(), CLUSTER_PARAMS.getServerId(), CLUSTER_PARAMS.getDefaultSummary(), CLUSTER_PARAMS.getDocumentdbInfoConfig(), new SchemaInfo(List.of(schema.build()), List.of())), - MockDispatcher.create(Collections.singletonList(new Node(CLUSTER, 0, "host0", 0)))); + MockDispatcher.create(List.of(new Node(CLUSTER, 0, "host0", 0)))); Query q = new Query("?query=foo"); Result result = doSearch(backend, q, 0, 10); assertFalse(backend.summaryNeedsQuery(q)); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java index f8fe0c92020..d2d5d124297 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.prelude.query; import static org.junit.jupiter.api.Assertions.*; import java.nio.ByteBuffer; -import java.util.Arrays; +import java.util.List; import java.util.ListIterator; import java.util.NoSuchElementException; import java.util.regex.PatternSyntaxException; @@ -211,7 +211,7 @@ public class ItemsCommonStuffTestCase { w.setConnectivity(v, 1.0); String expected = "puppy"; String expected2 = "kvalp"; - EquivItem e = new EquivItem(w, Arrays.asList(expected, expected2)); + EquivItem e = new EquivItem(w, List.of(expected, expected2)); assertEquals(1.0, e.getConnectivity(), 1e-9); assertSame(v, e.getConnectedItem()); assertEquals(expected, ((WordItem) e.getItem(1)).getWord()); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index fbede4613da..0b12cf99396 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -13,8 +13,8 @@ import com.yahoo.prelude.query.parser.Tokenizer; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Set; import static com.yahoo.prelude.query.parser.Token.Kind.COLON; import static com.yahoo.prelude.query.parser.Token.Kind.COMMA; @@ -283,7 +283,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts facts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + IndexFacts.Session session = facts.newSession(Set.of(), Set.of()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session); // tokenizer.print(); @@ -328,7 +328,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + IndexFacts.Session session = facts.newSession(Set.of(), Set.of()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -365,7 +365,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + IndexFacts.Session session = facts.newSession(Set.of(), Set.of()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -402,7 +402,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); + IndexFacts.Session session = facts.newSession(Set.of(), Set.of()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -439,7 +439,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts indexFacts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session facts = indexFacts.newSession(Collections.emptySet(), Collections.emptySet()); + IndexFacts.Session facts = indexFacts.newSession(Set.of(), Set.of()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java index f5dc598485a..475b8ddc02a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/textualrepresentation/test/TextualQueryRepresentationTestCase.java @@ -5,9 +5,9 @@ import java.io.BufferedReader; import java.io.FileReader; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import com.yahoo.prelude.query.Item; @@ -29,7 +29,7 @@ public class TextualQueryRepresentationTestCase { example; } - private class MockItem extends Item { + private static class MockItem extends Item { private final String name; @Override @@ -71,7 +71,7 @@ public class TextualQueryRepresentationTestCase { Map<Integer, Object> exampleMap = new HashMap<>(); exampleMap.put(1, "one"); exampleMap.put(2, "two"); - exampleMap.put(3, Arrays.asList('x', 'y', 'z')); + exampleMap.put(3, List.of('x', 'y', 'z')); discloser.addProperty("01", null); discloser.addProperty("02", "a string."); @@ -79,9 +79,9 @@ public class TextualQueryRepresentationTestCase { discloser.addProperty("04", true); discloser.addProperty("05", ExampleEnum.example); discloser.addProperty("06", new int[]{1, 2, 3}); - discloser.addProperty("07", Arrays.asList('x', 'y', 'z')); + discloser.addProperty("07", List.of('x', 'y', 'z')); discloser.addProperty("08", new ArrayList()); - discloser.addProperty("09", new HashSet(Arrays.asList(1, 2, 3))); + discloser.addProperty("09", new HashSet(List.of(1, 2, 3))); discloser.addProperty("10", exampleMap); discloser.setValue("example-value: \"12\""); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 4ce1cd5a10d..4b7c7e592cf 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.searcher.test; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -286,7 +285,7 @@ public class BlendingSearcherTestCase { assertEquals(1, cr.getConcreteHitCount()); com.yahoo.search.result.ErrorHit errorHit = cr.hits().getErrorHit(); Iterator errorIterator = errorHit.errorIterator(); - List<String> errorList = Arrays.asList("Source 'a': No backends in service. Try later", + List<String> errorList = List.of("Source 'a': No backends in service. Try later", "Source 'b': 2: Request too large"); String a = errorIterator.next().toString(); assertTrue(errorList.contains(a), a); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 8ced7d3895e..152056dfb72 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -28,11 +28,10 @@ import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.stream.StreamSupport; @@ -203,10 +202,10 @@ public class InterleavedSearchInvokerTest { } private static final double DELTA = 0.000000000001; - private static final List<Double> A5 = Arrays.asList(11.0,8.5,7.5,3.0,2.0); - private static final List<Double> B5 = Arrays.asList(9.0,8.0,7.0,6.0,1.0); - private static final List<Double> A5Aux = Arrays.asList(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0); - private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); + private static final List<Double> A5 = List.of(11.0,8.5,7.5,3.0,2.0); + private static final List<Double> B5 = List.of(9.0,8.0,7.0,6.0,1.0); + private static final List<Double> A5Aux = List.of(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0); + private static final List<Double> B5Aux = List.of(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK, Group group) throws IOException { try (InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5, group)) { @@ -356,7 +355,7 @@ public class InterleavedSearchInvokerTest { .addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3)))); invokers.add(new MockInvoker(0).setHits(List.of(new GroupingListHit(List.of(grouping2))))); - try (InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, hitEstimator, dispatchConfig, new Group(0, List.of()), Collections.emptySet())) { + try (InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, hitEstimator, dispatchConfig, new Group(0, List.of()), Set.of())) { invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); Result result = invoker.search(query); @@ -373,7 +372,7 @@ public class InterleavedSearchInvokerTest { List<SearchInvoker> invokers = new ArrayList<>(); invokers.add(createInvoker(a, 0)); invokers.add(createInvoker(b, 1)); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, hitEstimator, dispatchConfig, group, Collections.emptySet()); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, hitEstimator, dispatchConfig, group, Set.of()); invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); return invoker; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 8b6b3c4d13a..3053fe7d730 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.Test; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -238,7 +237,7 @@ public class LoadBalancerTest { } private GroupStatus newGroupStatus(int id) { - Group dummyGroup = new Group(id, Collections.emptyList()) { + Group dummyGroup = new Group(id, List.of()) { @Override public boolean hasSufficientCoverage() { return true; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/ProtobufSerializationTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/ProtobufSerializationTest.java index be8f99a4ef4..6459d67480d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/ProtobufSerializationTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/ProtobufSerializationTest.java @@ -14,7 +14,6 @@ import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -66,7 +65,7 @@ public class ProtobufSerializationTest { builder.setTimeout(0); var hit = new FastHit(); hit.setGlobalId(new GlobalId(IdString.createIdString("id:ns:type::id")).getRawId()); - var bytes = ProtobufSerialization.serializeDocsumRequest(builder, Collections.singletonList(hit)); + var bytes = ProtobufSerialization.serializeDocsumRequest(builder, List.of(hit)); assertEquals(56, bytes.length); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 5223eb0b06b..1b36c2b8151 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -12,7 +12,6 @@ import com.yahoo.search.result.ErrorMessage; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -46,7 +45,7 @@ public class SearchClusterTest { List<AtomicInteger> pingCounts; State(String clusterId, int nodesPergroup, String ... nodeNames) { - this(clusterId, nodesPergroup, Arrays.asList(nodeNames)); + this(clusterId, nodesPergroup, List.of(nodeNames)); } State(String clusterId, int nodesPerGroup, List<String> nodeNames) { diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java index 4d3e3c18e0b..1ac6dcbe1b6 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java @@ -23,7 +23,6 @@ import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -339,7 +338,7 @@ public class FederationSearcherTest { @Override public Collection<FederationTarget<String>> getTargets(Query query, ChainRegistry<Searcher> searcherChainRegistry) { - return Arrays.asList(createTarget(1), createTarget(2)); + return List.of(createTarget(1), createTarget(2)); } private FederationTarget<String> createTarget(int number) { diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java index d9046075f38..e5bbb48e807 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SearchChainResolverTestCase.java @@ -13,8 +13,8 @@ import java.util.Iterator; import java.util.SortedSet; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; /** * @author Tony Vaagenes @@ -59,37 +59,33 @@ public class SearchChainResolverTestCase { @Test void require_error_message_for_invalid_source() { - try { - resolve("no-such-source"); - fail("Expected exception."); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'no-such-source'.", e.getMessage()); - } + var result = resolve("no-such-source"); + assertEquals("Could not resolve source ref 'no-such-source'.", result.errorMsg()); } @Test - void lookup_search_chain() throws Exception { - SearchChainInvocationSpec res = resolve(searchChainId.getName()); + void lookup_search_chain() { + SearchChainInvocationSpec res = resolve(searchChainId.getName()).invocationSpec(); assertEquals(searchChainId, res.searchChainId); } //TODO: TVT: @Test() - public void lookup_provider() throws Exception { - SearchChainInvocationSpec res = resolve(providerId.getName()); + public void lookup_provider() { + SearchChainInvocationSpec res = resolve(providerId.getName()).invocationSpec(); assertEquals(providerId, res.provider); assertNull(res.source); assertEquals(providerId, res.searchChainId); } @Test - void lookup_source() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName()); + void lookup_source() { + SearchChainInvocationSpec res = resolve(sourceId.getName()).invocationSpec(); assertIsSourceInProvider(res); } @Test - void lookup_source_search_chain_directly() throws Exception { - SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + void lookup_source_search_chain_directly() { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()).invocationSpec(); assertIsSourceInProvider(res); } @@ -100,8 +96,8 @@ public class SearchChainResolverTestCase { } @Test - void lookup_source_for_provider2() throws Exception { - SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + void lookup_source_for_provider2() { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()).invocationSpec(); assertEquals(provider2Id, res.provider); assertEquals(sourceId, res.source); assertEquals(sourceChainInProvider2Id, res.searchChainId); @@ -126,22 +122,24 @@ public class SearchChainResolverTestCase { return new PropertyMap(); } - private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification) { return resolve(sourceSpecification, emptySourceToProviderMap()); } - private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) - throws UnresolvedSearchChainException { + private ResolveResult resolve(String sourceSpecification, String providerSpecification) { Properties sourceToProviderMap = emptySourceToProviderMap(); sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); return resolve(sourceSpecification, sourceToProviderMap); } - private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) - throws UnresolvedSearchChainException { - SearchChainInvocationSpec res = searchChainResolver.resolve( + private ResolveResult resolve(String sourceSpecification, Properties sourceToProviderMap) { + ResolveResult res = searchChainResolver.resolve( ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); - assertEquals(federationOptions, res.federationOptions); + if (res.invocationSpec() != null) { + assertEquals(federationOptions, res.invocationSpec().federationOptions); + } else { + assertNotNull(res.errorMsg()); + } return res; } diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java index b32135afc94..95262937c01 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/SourceRefResolverTestCase.java @@ -3,21 +3,16 @@ package com.yahoo.search.federation.sourceref; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.IndexModel; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import static com.yahoo.search.federation.sourceref.SearchChainResolverTestCase.emptySourceToProviderMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; /** * Test for SourceRefResolver. @@ -47,49 +42,38 @@ public class SourceRefResolverTestCase { @Test void lookup_search_chain() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve(cluster1); + List<ResolveResult> searchChains = resolve(cluster1); assertEquals(1, searchChains.size()); assertTrue(searchChainIds(searchChains).contains(cluster1)); } @Test void lookup_search_chains_for_document1() throws Exception { - Set<SearchChainInvocationSpec> searchChains = resolve("document1"); + List<ResolveResult> searchChains = resolve("document1"); assertEquals(2, searchChains.size()); assertTrue(searchChainIds(searchChains).containsAll(List.of(cluster1, cluster2))); } @Test void error_when_document_gives_cluster_without_matching_search_chain() { - try { - resolve("document3"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Failed to resolve cluster search chain 'cluster3' " + - "when using source ref 'document3' as a document name.", - e.getMessage()); - } + List<ResolveResult> result = resolve("document3"); + + assertEquals("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.", + result.get(0).errorMsg()); } @Test void error_when_no_document_or_search_chain() { - try { - resolve("document4"); - fail("Expected exception"); - } catch (UnresolvedSearchChainException e) { - assertEquals("Could not resolve source ref 'document4'.", e.getMessage()); - } + List<ResolveResult> results = resolve("document4"); + assertEquals("Could not resolve source ref 'document4'.", results.get(0).errorMsg()); } - private List<String> searchChainIds(Set<SearchChainInvocationSpec> searchChains) { - List<String> names = new ArrayList<>(); - for (SearchChainInvocationSpec searchChain : searchChains) { - names.add(searchChain.searchChainId.stringValue()); - } - return names; + private List<String> searchChainIds(Collection<ResolveResult> searchChains) { + return searchChains.stream().map(r -> r.invocationSpec().searchChainId.stringValue()).toList(); } - private Set<SearchChainInvocationSpec> resolve(String documentName) throws UnresolvedSearchChainException { + private List<ResolveResult> resolve(String documentName) { return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap()); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java index 76a1a71f6ed..540ecfa6e12 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java @@ -9,7 +9,6 @@ import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import java.util.TimeZone; @@ -22,12 +21,12 @@ public class GroupingQueryParserTestCase { @Test void requireThatNoRequestIsSkipped() { - assertEquals(Collections.emptyList(), executeQuery(null, null, null)); + assertEquals(List.of(), executeQuery(null, null, null)); } @Test void requireThatEmptyRequestIsSkipped() { - assertEquals(Collections.emptyList(), executeQuery("", null, null)); + assertEquals(List.of(), executeQuery("", null, null)); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java index 97eed95946f..3af2614b6fb 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java @@ -11,8 +11,6 @@ import org.junit.jupiter.api.Test; import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -34,7 +32,7 @@ public class GroupingRequestTestCase { } }; req.continuations().add(foo); - assertEquals(Arrays.asList(foo), req.continuations()); + assertEquals(List.of(foo), req.continuations()); req.continuations().clear(); assertTrue(req.continuations().isEmpty()); @@ -121,7 +119,7 @@ public class GroupingRequestTestCase { @Test void requireThatGetRequestsReturnsAllRequests() { Query query = new Query(); - assertEquals(Collections.emptyList(), query.getSelect().getGrouping()); + assertEquals(List.of(), query.getSelect().getGrouping()); GroupingRequest foo = GroupingRequest.newInstance(query); assertEquals(List.of(foo), query.getSelect().getGrouping()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java index 800faa4c0d1..19fd4234cfc 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.search.grouping.request; import org.junit.jupiter.api.Test; import java.text.ChoiceFormat; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -72,30 +71,30 @@ public class BucketResolverTestCase { @Test void testBucketType() { - checkPushFail(Arrays.asList((ConstantValue) new StringValue("a"), new LongValue(1L)), + checkPushFail(List.of((ConstantValue) new StringValue("a"), new LongValue(1L)), "Bucket type mismatch, expected 'StringValue' got 'LongValue'."); - checkPushFail(Arrays.asList((ConstantValue) new StringValue("a"), new DoubleValue(1.0)), + checkPushFail(List.of((ConstantValue) new StringValue("a"), new DoubleValue(1.0)), "Bucket type mismatch, expected 'StringValue' got 'DoubleValue'."); - checkPushFail(Arrays.asList((ConstantValue) new LongValue(1L), new StringValue("a")), + checkPushFail(List.of((ConstantValue) new LongValue(1L), new StringValue("a")), "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); - checkPushFail(Arrays.asList((ConstantValue) new LongValue(1L), new DoubleValue(1.0)), + checkPushFail(List.of((ConstantValue) new LongValue(1L), new DoubleValue(1.0)), "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); - checkPushFail(Arrays.asList((ConstantValue) new DoubleValue(1.0), new StringValue("a")), + checkPushFail(List.of((ConstantValue) new DoubleValue(1.0), new StringValue("a")), "Bucket type mismatch, expected 'DoubleValue' got 'StringValue'."); - checkPushFail(Arrays.asList((ConstantValue) new DoubleValue(1.0), new LongValue(1L)), + checkPushFail(List.of((ConstantValue) new DoubleValue(1.0), new LongValue(1L)), "Bucket type mismatch, expected 'DoubleValue' got 'LongValue'."); - checkPushFail(Arrays.asList((ConstantValue) new InfiniteValue(new Infinite(true)), new InfiniteValue(new Infinite(false))), + checkPushFail(List.of((ConstantValue) new InfiniteValue(new Infinite(true)), new InfiniteValue(new Infinite(false))), "Bucket type mismatch, cannot both be infinity."); } @Test void testBucketOrder() { - checkPushFail(Arrays.asList((ConstantValue) new LongValue(2L), new LongValue(1L)), + checkPushFail(List.of((ConstantValue) new LongValue(2L), new LongValue(1L)), "Bucket to-value can not be less than from-value."); - checkPushFail(Arrays.asList((ConstantValue) new DoubleValue(2.0), new DoubleValue(1.0)), + checkPushFail(List.of((ConstantValue) new DoubleValue(2.0), new DoubleValue(1.0)), "Bucket to-value can not be less than from-value."); - checkPushFail(Arrays.asList((ConstantValue) new StringValue("b"), new StringValue("a")), + checkPushFail(List.of((ConstantValue) new StringValue("b"), new StringValue("a")), "Bucket to-value can not be less than from-value."); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java index 85f19e0b19a..daed437e93f 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.*; * @author Einar M R Rosenvinge */ public class MathFunctionsTestCase { + private static final DoubleValue ZERO = new DoubleValue(0.0); @Test void testMathFunctions() { @@ -37,27 +38,27 @@ public class MathFunctionsTestCase { assertSame(MathFunctions.Function.create(19), MathFunctions.Function.HYPOT); assertSame(MathFunctions.Function.create(20), MathFunctions.Function.FLOOR); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.EXP, null, null) instanceof MathExpFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.POW, null, null) instanceof MathPowFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG, null, null) instanceof MathLogFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG1P, null, null) instanceof MathLog1pFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG10, null, null) instanceof MathLog10Function); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.SIN, null, null) instanceof MathSinFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ASIN, null, null) instanceof MathASinFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.COS, null, null) instanceof MathCosFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ACOS, null, null) instanceof MathACosFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.TAN, null, null) instanceof MathTanFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ATAN, null, null) instanceof MathATanFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.SQRT, null, null) instanceof MathSqrtFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.SINH, null, null) instanceof MathSinHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ASINH, null, null) instanceof MathASinHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.COSH, null, null) instanceof MathCosHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ACOSH, null, null) instanceof MathACosHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.TANH, null, null) instanceof MathTanHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.ATANH, null, null) instanceof MathATanHFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.CBRT, null, null) instanceof MathCbrtFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.HYPOT, null, null) instanceof MathHypotFunction); - assertTrue(MathFunctions.newInstance(MathFunctions.Function.FLOOR, null, null) instanceof MathFloorFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.EXP, ZERO, ZERO) instanceof MathExpFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.POW, ZERO, ZERO) instanceof MathPowFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG, ZERO, ZERO) instanceof MathLogFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG1P, ZERO, ZERO) instanceof MathLog1pFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.LOG10, ZERO, ZERO) instanceof MathLog10Function); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.SIN, ZERO, ZERO) instanceof MathSinFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ASIN, ZERO, ZERO) instanceof MathASinFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.COS, ZERO, ZERO) instanceof MathCosFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ACOS, ZERO, ZERO) instanceof MathACosFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.TAN, ZERO, ZERO) instanceof MathTanFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ATAN, ZERO, ZERO) instanceof MathATanFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.SQRT, ZERO, ZERO) instanceof MathSqrtFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.SINH, ZERO, ZERO) instanceof MathSinHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ASINH, ZERO, ZERO) instanceof MathASinHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.COSH, ZERO, ZERO) instanceof MathCosHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ACOSH, ZERO, ZERO) instanceof MathACosHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.TANH, ZERO, ZERO) instanceof MathTanHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.ATANH, ZERO, ZERO) instanceof MathATanHFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.CBRT, ZERO, ZERO) instanceof MathCbrtFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.HYPOT, ZERO, ZERO) instanceof MathHypotFunction); + assertTrue(MathFunctions.newInstance(MathFunctions.Function.FLOOR, ZERO, ZERO) instanceof MathFloorFunction); } } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java index babeebdf8c1..71ffc5e928d 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.search.grouping.request; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -42,8 +41,8 @@ public class RawBufferTestCase { @Test void requireThatToStringWorks() { - assertToString(Arrays.asList("a".getBytes()[0], "b".getBytes()[0]), "{97,98}"); - assertToString(Arrays.asList((byte) 2, (byte) 6), "{2,6}"); + assertToString(List.of("a".getBytes()[0], "b".getBytes()[0]), "{97,98}"); + assertToString(List.of((byte) 2, (byte) 6), "{2,6}"); } public void assertToString(List<Byte> data, String expected) { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java index 0b6ab49867b..24049264d4b 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.search.grouping.request; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -17,7 +17,7 @@ public class RequestTestCase { GroupingOperation op = new AllOperation() .setGroupBy(new AttributeValue("foo")) .addOrderBy(new CountAggregator()) - .addChildren(Arrays.asList(new AllOperation(), new EachOperation())) + .addChildren(List.of(new AllOperation(), new EachOperation())) .addChild(new EachOperation() .addOutput(new CountAggregator()) .addOutput(new MinAggregator(new AttributeValue("bar"))) diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java index 2471acfb115..4d03c4aff95 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -4,7 +4,6 @@ package com.yahoo.search.grouping.request.parser; import com.yahoo.search.grouping.request.GroupingOperation; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -51,7 +50,7 @@ public class GroupingParserBenchmarkTest { } private static List<String> getInputs() { - return Arrays.asList( + return List.of( " all(group(foo)each(output(max(bar))))", "all( group(foo)each(output(max(bar))))", "all(group( foo)each(output(max(bar))))", diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index e78ebfbd5af..01963e94546 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.search.yql.YqlParser; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -29,7 +28,7 @@ public class GroupingParserTestCase { @Test void requireThatMathAllowsWhitespace() { - for (String op : Arrays.asList("+", " +", " + ", "+ ", + for (String op : List.of("+", " +", " + ", "+ ", "-", " -", " - ", "- ", "*", " *", " * ", "* ", "/", " /", " / ", "/ ", @@ -64,7 +63,7 @@ public class GroupingParserTestCase { @Test void requireThatTokenImagesAreNotReservedWords() { - List<String> images = Arrays.asList("acos", + List<String> images = List.of("acos", "acosh", "accuracy", "add", @@ -489,7 +488,7 @@ public class GroupingParserTestCase { @Test void testMisc() { - for (String fnc : Arrays.asList("time.date", + for (String fnc : List.of("time.date", "time.dayofmonth", "time.dayofweek", "time.dayofyear", @@ -634,7 +633,7 @@ public class GroupingParserTestCase { actual.add(operation.toString()); } if (expectedOperations.length > 0) { - assertEquals(Arrays.asList(expectedOperations), actual); + assertEquals(List.of(expectedOperations), actual); } // make sure that operation does not mutate through toString() -> fromString() @@ -656,7 +655,7 @@ public class GroupingParserTestCase { actual.add(step.getOperation().toString()); } if (expectedOperations.length > 0) { - assertEquals(Arrays.asList(expectedOperations), actual); + assertEquals(List.of(expectedOperations), actual); } } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java index 9e8fcb0ea21..c17bc985aef 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java @@ -28,7 +28,6 @@ import com.yahoo.searchlib.aggregation.hll.SparseSketch; import com.yahoo.searchlib.expression.StringResultNode; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Queue; @@ -105,7 +104,7 @@ public class FlatteningSearcherTestCase { } private static Execution newExecution(Searcher... searchers) { - return new Execution(new SearchChain(new ComponentId("foo"), Arrays.asList(searchers)), + return new Execution(new SearchChain(new ComponentId("foo"), List.of(searchers)), Execution.Context.createContextStub()); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index ef2ef9724a9..780066d0afe 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -35,7 +35,6 @@ import com.yahoo.searchlib.expression.StringResultNode; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -180,7 +179,7 @@ public class GroupingExecutorTestCase { .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("common")).addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3))) ); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grpA), null, query), new GroupingListHit(List.of(grpB), null, query)))); Group grp = req.getResultGroup(exec.search(query)); @@ -213,7 +212,7 @@ public class GroupingExecutorTestCase { .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("unexpected")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(96)).setTag(3))) ); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grpExpected), null, query), new GroupingListHit(List.of(grpUnexpected), null, query)))); Group grp = req.getResultGroup(exec.search(query)); @@ -244,7 +243,7 @@ public class GroupingExecutorTestCase { .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit())) )); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grp0), null, query), new GroupingListHit(List.of(grp1), null, query))), new FillRequestThrower()); @@ -284,7 +283,7 @@ public class GroupingExecutorTestCase { new HitsAggregationResult(1, "bar") .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grp0), null, query), new GroupingListHit(List.of(grp1), null, query))), new FillErrorProvider()); @@ -310,7 +309,7 @@ public class GroupingExecutorTestCase { .addAggregationResult(new CountAggregationResult(2)) .addOrderBy(new AggregationRefNode(0), true))); Result res = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grp), null, query), new GroupingListHit(List.of(grp), null, query)))).search(query); @@ -339,7 +338,7 @@ public class GroupingExecutorTestCase { ErrorProvider err = new ErrorProvider(1); Execution exec = newExecution(new GroupingExecutor(), err, - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grp0), null, query), new GroupingListHit(List.of(grp1), null, query)))); Result res = exec.search(query); @@ -350,7 +349,7 @@ public class GroupingExecutorTestCase { err = new ErrorProvider(0); exec = newExecution(new GroupingExecutor(), err, - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(grp0), null, query), new GroupingListHit(List.of(grp1), null, query)))); res = exec.search(query); @@ -389,9 +388,9 @@ public class GroupingExecutorTestCase { )); SummaryMapper sm = new SummaryMapper(); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( - new GroupingListHit(Arrays.asList(pass0A, pass0B), null, query), - new GroupingListHit(Arrays.asList(pass1A, pass1B), null, query))), + new ResultProvider(List.of( + new GroupingListHit(List.of(pass0A, pass0B), null, query), + new GroupingListHit(List.of(pass1A, pass1B), null, query))), sm); exec.fill(exec.search(query), "default"); assertEquals(2, sm.hitsBySummary.size()); @@ -433,7 +432,7 @@ public class GroupingExecutorTestCase { new HitsAggregationResult(1, ExpressionConverter.DEFAULT_SUMMARY_NAME) .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(pass0), null, query), new GroupingListHit(List.of(pass1), null, query)))); Result res = exec.search(query); @@ -468,7 +467,7 @@ public class GroupingExecutorTestCase { QueryMapper qm = new QueryMapper(); Execution exec = newExecution(new GroupingExecutor(), - new ResultProvider(Arrays.asList(pass0, pass1)), + new ResultProvider(List.of(pass0, pass1)), qm); exec.fill(exec.search(queryA)); assertEquals(1, qm.hitsByQuery.size()); @@ -607,7 +606,7 @@ public class GroupingExecutorTestCase { } private static Execution newExecution(Searcher... searchers) { - return new Execution(new SearchChain(new ComponentId("foo"), Arrays.asList(searchers)), + return new Execution(new SearchChain(new ComponentId("foo"), List.of(searchers)), Execution.Context.createContextStub()); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java index 6230899ec49..7e049e44f4d 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -36,7 +36,6 @@ import com.yahoo.searchlib.expression.StringResultNode; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -916,10 +915,11 @@ public class ResultBuilderTestCase { assertOutput(test); } + private static void assertOutput(ResultTest test) { RequestBuilder reqBuilder = new RequestBuilder(REQUEST_ID); reqBuilder.setRootOperation(GroupingOperation.fromString(test.request)); - reqBuilder.addContinuations(Arrays.asList(test.continuation)); + reqBuilder.addContinuations(test.getContinuations()); reqBuilder.build(); assertEquals(reqBuilder.getRequestList().size(), test.result.size()); @@ -973,6 +973,9 @@ public class ResultBuilderTestCase { String expectedException; OutputWriter outputWriter; Continuation continuation; + List<Continuation> getContinuations() { + return continuation != null ? List.of(continuation) : List.of(); + } } private static interface OutputWriter { diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java index 1869e0e4bf0..78875b5b831 100644 --- a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java @@ -100,7 +100,7 @@ public class PageTemplateSearcherTestCase { { // Specifying two templates as a list, should override the page.id setting Query query = new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); - query.properties().set("page.idList", Arrays.asList("oneSource", "threeSources")); + query.properties().set("page.idList", List.of("oneSource", "threeSources")); Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); assertSources("source1 source2 source3", "source1 source2 source3", result); } @@ -172,11 +172,11 @@ public class PageTemplateSearcherTestCase { } private void assertSources(String expectedQuerySourceString,String expectedResultSourceString,Result result) { - Set<String> expectedQuerySources=new HashSet<>(Arrays.asList(expectedQuerySourceString.split(" "))); + Set<String> expectedQuerySources=new HashSet<>(List.of(expectedQuerySourceString.split(" "))); assertEquals(expectedQuerySources,result.getQuery().getModel().getSources()); - Set<String> expectedResultSources=new HashSet<>(Arrays.asList(expectedResultSourceString.split(" "))); - for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) { + Set<String> expectedResultSources=new HashSet<>(List.of(expectedResultSourceString.split(" "))); + for (String sourceName : List.of("source1 source2 source3".split(" "))) { if (expectedResultSources.contains(sourceName)) assertNotNull(result.hits().get(sourceName),"Result contains '" + sourceName + "'"); else @@ -189,7 +189,7 @@ public class PageTemplateSearcherTestCase { @Override public Result search(Query query,Execution execution) { Result result=new Result(query); - for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) + for (String sourceName : List.of("source1 source2 source3".split(" "))) if (query.getModel().getSources().isEmpty() || query.getModel().getSources().contains(sourceName)) result.hits().add(createSource(sourceName)); return result; diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java index 49eaa9b3a89..759181a2ce7 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -82,7 +82,7 @@ public class SortingTestCase { private void requireThatChineseHasCorrectRules(Collator col) { final int reorderCodes [] = {UScript.HAN}; assertEquals("15.1.0.0", col.getUCAVersion().toString()); - assertEquals("153.121.44.8", col.getVersion().toString()); + assertEquals("153.121.45.0", col.getVersion().toString()); assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); assertNotEquals("", ((RuleBasedCollator) col).getRules()); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 20043f23256..58f1fa62fe6 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -20,7 +20,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.trace.TraceNode; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -125,7 +124,7 @@ public class QueryProfileTestCase { far.set("a.far", "a.far", null); CompiledQueryProfile cbarn = barn.compile(null); - assertSameObjects(cbarn, "a", Arrays.asList("mormor", "far", "barn")); + assertSameObjects(cbarn, "a", List.of("mormor", "far", "barn")); assertEquals("b.mor", cbarn.get("b.mor")); assertEquals("b.far", cbarn.get("b.far")); diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java index 467a0b0845c..266cb4daf67 100644 --- a/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java @@ -4,8 +4,8 @@ package com.yahoo.search.query.properties; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import com.yahoo.processing.request.properties.PropertyMap; import org.junit.jupiter.api.Test; @@ -34,7 +34,7 @@ public class SubPropertiesTestCase { assertEquals("1", sub.get("e")); assertEquals(2, sub.get("f")); assertNull(sub.get("d")); - assertEquals(new HashSet<>(Arrays.asList("e", "f")), sub.listProperties("").keySet()); + assertEquals(new HashSet<>(List.of("e", "f")), sub.listProperties("").keySet()); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java index 42dcab41015..793a907b5d1 100644 --- a/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.search.query.properties.test; import com.yahoo.processing.request.properties.PropertyMap; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -22,8 +21,8 @@ public class PropertyMapTestCase { map.set("nonclonable", new NonClonableObject()); map.set("clonableArray", new ClonableObject[]{new ClonableObject()}); map.set("nonclonableArray", new NonClonableObject[]{new NonClonableObject()}); - map.set("clonableList", Collections.singletonList(new ClonableObject())); - map.set("nonclonableList", Collections.singletonList(new NonClonableObject())); + map.set("clonableList", List.of(new ClonableObject())); + map.set("nonclonableList", List.of(new NonClonableObject())); assertNotNull(map.get("clonable")); assertNotNull(map.get("nonclonable")); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java index 2cd43257b12..89a62abddcc 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java @@ -10,8 +10,8 @@ import org.junit.jupiter.api.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.Arrays; import java.util.LinkedHashSet; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -91,8 +91,8 @@ public class ModelTestCase { Model sr = new Model(q); sr.setRestrict("music, cheese,other"); sr.setSources("cluster1"); - assertEquals(sr.getSources(), new LinkedHashSet<>(Arrays.asList(new String[]{"cluster1"}))); - assertEquals(sr.getRestrict(), new LinkedHashSet<>(Arrays.asList(new String[]{"cheese", "music", "other"}))); + assertEquals(sr.getSources(), new LinkedHashSet<>(List.of(new String[]{"cluster1"}))); + assertEquals(sr.getRestrict(), new LinkedHashSet<>(List.of(new String[]{"cheese", "music", "other"}))); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java index 732fdd7dcbb..7bfbeef4e23 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java @@ -36,7 +36,6 @@ public class RankFeaturesTestCase { } @Test - @SuppressWarnings("deprecation") void requireThatRankFeaturesUsingDoubleAndDoubleToStringEncodeTheSameWay() { RankFeatures withDouble = new RankFeatures(new Ranking(new Query())); withDouble.put("query(myDouble)", 3.8); @@ -68,20 +67,12 @@ public class RankFeaturesTestCase { TensorType type = new TensorType.Builder().mapped("x").mapped("y").mapped("z").build(); Tensor tensor1 = Tensor.from(type, "{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); Tensor tensor2 = Tensor.from(type, "{ {x:a, y:b, z:c}:5.0 }"); - assertTensorEncodingAndDecoding(type, Arrays.asList( + assertTensorEncodingAndDecoding(type, List.of( new Entry("query(tensor1)", "tensor1", tensor1), new Entry("$tensor2", "tensor2", tensor2))); } - private static class Entry { - final String key; - final String normalizedKey; - final Tensor tensor; - Entry(String key, String normalizedKey, Tensor tensor) { - this.key = key; - this.normalizedKey = normalizedKey; - this.tensor = tensor; - } + private record Entry(String key, String normalizedKey, Tensor tensor) { } private static void assertTensorEncodingAndDecoding(TensorType type, List<Entry> entries) { @@ -97,7 +88,7 @@ public class RankFeaturesTestCase { } private static void assertTensorEncodingAndDecoding(TensorType type, String key, String normalizedKey, Tensor tensor) { - assertTensorEncodingAndDecoding(type, Arrays.asList(new Entry(key, normalizedKey, tensor))); + assertTensorEncodingAndDecoding(type, List.of(new Entry(key, normalizedKey, tensor))); } private static RankProperties createRankPropertiesWithTensors(List<Entry> entries) { diff --git a/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java index 39b202daf1e..b37e1d5551b 100644 --- a/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java +++ b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java @@ -35,7 +35,7 @@ public class GlobalPhaseRerankHitsImplTest { } } static FunEvalSpec makeConstSpec(double constValue) { - return new FunEvalSpec(() -> new EvalSum(constValue), Collections.emptyList(), Collections.emptyList()); + return new FunEvalSpec(() -> new EvalSum(constValue), List.of(), List.of()); } static FunEvalSpec makeSumSpec(List<String> fromQuery, List<String> fromMF) { List<MatchFeatureInput> mfList = new ArrayList<>(); @@ -175,15 +175,15 @@ public class GlobalPhaseRerankHitsImplTest { } @Test void partialRerankWithRescaling() { var setup = setup().rerank(2).eval(makeConstSpec(3.0)).build(); - var query = makeQuery(Collections.emptyList()); + var query = makeQuery(List.of()); var result = makeResult(query, List.of(hit("a", 3), hit("b", 4), hit("c", 5), hit("d", 6))); var expect = Expect.make(List.of(hit("a", 1), hit("b", 2), hit("c", 3), hit("d", 3))); GlobalPhaseRanker.rerankHitsImpl(setup, query, result); expect.verifyScores(result); } @Test void matchFeaturesCanBePartiallyHidden() { - var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("public_value", "private_value"))).hide("private_value").build(); - var query = makeQuery(Collections.emptyList()); + var setup = setup().eval(makeSumSpec(List.of(), List.of("public_value", "private_value"))).hide("private_value").build(); + var query = makeQuery(List.of()); var factory = new HitFactory(List.of("public_value", "private_value")); var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("public_value", 2), value("private_value", 3))), factory.create("b", 2, List.of(value("public_value", 5), value("private_value", 7))))); @@ -194,8 +194,8 @@ public class GlobalPhaseRerankHitsImplTest { verifyDoesNotHaveMF(result, "private_value"); } @Test void matchFeaturesCanBeRemoved() { - var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("private_value"))).hide("private_value").build(); - var query = makeQuery(Collections.emptyList()); + var setup = setup().eval(makeSumSpec(List.of(), List.of("private_value"))).hide("private_value").build(); + var query = makeQuery(List.of()); var factory = new HitFactory(List.of("private_value")); var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("private_value", 3))), factory.create("b", 2, List.of(value("private_value", 7))))); @@ -227,7 +227,7 @@ public class GlobalPhaseRerankHitsImplTest { verifyHasMF(result, "bar"); } @Test void queryFeaturesCanBeDefaultValues() { - var setup = setup().eval(makeSumSpec(List.of("foo", "bar"), Collections.emptyList())) + var setup = setup().eval(makeSumSpec(List.of("foo", "bar"), List.of())) .addDefault("query(bar)", Tensor.from(5.0)).build(); var query = makeQuery(List.of(value("query(foo)", 7))); var result = makeResult(query, List.of(hit("a", 1))); @@ -236,7 +236,7 @@ public class GlobalPhaseRerankHitsImplTest { expect.verifyScores(result); } @Test void withNormalizer() { - var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("bar"))) + var setup = setup().eval(makeSumSpec(List.of(), List.of("bar"))) .addNormalizer(makeNormalizer("foo", List.of(115.0, 65.0, 55.0, 45.0, 15.0), makeSumSpec(List.of("x"), List.of("bar")))).build(); var query = makeQuery(List.of(value("query(x)", 5))); var factory = new HitFactory(List.of("bar")); diff --git a/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java index 298673a1ae7..d8f4f39c5b8 100644 --- a/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java @@ -5,7 +5,8 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import org.junit.jupiter.api.Test; -import java.util.Collections; + +import java.util.Set; import static org.junit.jupiter.api.Assertions.*; @@ -63,7 +64,7 @@ public class FillingTestCase { hits.add(hit1); hits.add(hit2); - assertEquals(Collections.emptySet(), hits.getFilled()); + assertEquals(Set.of(), hits.getFilled()); } @Test @@ -82,7 +83,7 @@ public class FillingTestCase { hits.add(hit2); hits.add(hit3); - assertEquals(Collections.singleton("summary1"), hits.getFilled()); + assertEquals(Set.of("summary1"), hits.getFilled()); } private Hit createNonFilled(String id) { diff --git a/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java index 5f8fbbd08df..42098051b82 100644 --- a/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java @@ -9,7 +9,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -165,14 +164,14 @@ public class HitGroupTestCase { Hit hit = new Hit("http://nalle.balle/1.html", 832); hit.setField("url", "http://nalle.balle/1.html"); hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hg.add(hit); } { Hit hit = new Hit("http://nalle.balle/2.html", 442); hit.setField("url", "http://nalle.balle/2.html"); hit.setField("clickurl", ""); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hg.add(hit); } assertFalse(hg.isFillable()); @@ -187,7 +186,7 @@ public class HitGroupTestCase { Hit hit = new Hit("http://nalle.balle/1.html", 832); hit.setField("url", "http://nalle.balle/1.html"); hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hit.setFillable(); hg.add(hit); } @@ -195,7 +194,7 @@ public class HitGroupTestCase { Hit hit = new Hit("http://nalle.balle/2.html", 442); hit.setField("url", "http://nalle.balle/2.html"); hit.setField("clickurl", ""); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hit.setFillable(); hg.add(hit); } @@ -211,14 +210,14 @@ public class HitGroupTestCase { Hit hit = new Hit("http://nalle.balle/1.html", 832); hit.setField("url", "http://nalle.balle/1.html"); hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hg.add(hit); } { Hit hit = new Hit("http://nalle.balle/2.html", 442); hit.setField("url", "http://nalle.balle/2.html"); hit.setField("clickurl", ""); - hit.setField("attributes", Arrays.asList("typevideo")); + hit.setField("attributes", List.of("typevideo")); hg.add(hit); } assertFalse(hg.isFillable()); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/AsyncExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/AsyncExecutionTestCase.java index e8a85e38e80..1bf04eb44fb 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/AsyncExecutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/AsyncExecutionTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.search.result.Hit; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -70,20 +69,20 @@ public class AsyncExecutionTestCase { void testWaitForAll() { Chain<Searcher> slowChain = new Chain<>( new ComponentId("slow"), - Arrays.asList(new Searcher[]{new WaitingSearcher("slow", 30000)} + List.of(new Searcher[]{new WaitingSearcher("slow", 30000)} ) ); Chain<Searcher> fastChain = new Chain<>( new ComponentId("fast"), - Arrays.asList(new Searcher[]{new SimpleSearcher()}) + List.of(new Searcher[]{new SimpleSearcher()}) ); FutureResult slowFuture = new AsyncExecution(slowChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); FutureResult fastFuture = new AsyncExecution(fastChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); fastFuture.get(); FutureResult [] reslist = new FutureResult[]{slowFuture, fastFuture}; - List<Result> results = AsyncExecution.waitForAll(Arrays.asList(reslist), 0); + List<Result> results = AsyncExecution.waitForAll(List.of(reslist), 0); //assertTrue(slowFuture.isCancelled()); assertTrue(fastFuture.isDone() && !fastFuture.isCancelled()); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java index 924aa7ae999..16e9adecdba 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.search.searchchain.config.test; import java.io.File; import java.io.IOException; -import java.util.Arrays; +import java.util.List; import com.yahoo.component.chain.dependencies.After; @@ -72,8 +72,8 @@ public class DependencyConfigTestCase { void test() { Dependencies dependencies = registry.getSearcherRegistry().getComponent(Searcher1.class.getName()).getDependencies(); - assertTrue(dependencies.provides().containsAll(Arrays.asList("P", "P1", "P2", Searcher1.class.getSimpleName()))); - assertTrue(dependencies.before().containsAll(Arrays.asList("B", "B1", "B2"))); - assertTrue(dependencies.after().containsAll(Arrays.asList("A", "A1", "A2"))); + assertTrue(dependencies.provides().containsAll(List.of("P", "P1", "P2", Searcher1.class.getSimpleName()))); + assertTrue(dependencies.before().containsAll(List.of("B", "B1", "B2"))); + assertTrue(dependencies.after().containsAll(List.of("A", "A1", "A2"))); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java index bfcde54d65b..fda7bec9d71 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java @@ -16,7 +16,7 @@ import com.yahoo.search.searchchain.SearchChainRegistry; import com.yahoo.search.searchchain.model.federation.FederationOptions; import org.junit.jupiter.api.Test; -import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -82,7 +82,7 @@ public class FutureDataTestCase { void testFutureData() throws InterruptedException, ExecutionException, TimeoutException { // Set up AsyncProviderSearcher futureDataSource = new AsyncProviderSearcher(); - Chain<Searcher> chain = new Chain<>(Collections.<Searcher>singletonList(futureDataSource)); + Chain<Searcher> chain = new Chain<>(List.of(futureDataSource)); // Execute Query query = new Query(); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java index 2f04b695774..87384a15979 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java @@ -5,7 +5,6 @@ import static com.yahoo.search.searchchain.test.SimpleSearchChain.searchChain; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -40,7 +39,7 @@ public class SearchChainTestCase { assertEquals("test", searchChain.getId().getName()); assertEquals(Version.emptyVersion, searchChain.getId().getVersion()); assertEquals(new Version(), searchChain.getId().getVersion()); - assertEqualMembers(Arrays.asList("one", "two"), searcherNames(searchChain.searchers())); + assertEqualMembers(List.of("one", "two"), searcherNames(searchChain.searchers())); } public List<String> searcherNames(Collection<Searcher> searchers) { diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java index de2fc46d803..833fd2c8657 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java @@ -2,7 +2,6 @@ package com.yahoo.search.searchchain.test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -38,7 +37,7 @@ public class SimpleSearchChain { @Override public Collection<ForkingSearcher.CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry) { - return Arrays.asList( + return List.of( new ForkingSearcher.CommentedSearchChain("Reason for forwarding to this search chain.", dummySearchChain()), new ForkingSearcher.CommentedSearchChain(null, dummySearchChain())); } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java index 036c41aab66..fb77073f68e 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -308,7 +307,7 @@ public class TraceTestCase { public Forker(boolean carryOverContext, boolean parallel, Searcher ... branches) { this.carryOverContext = carryOverContext; this.parallel = parallel; - this.branches = Arrays.asList(branches); + this.branches = List.of(branches); } @Override diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java index c4b8c9f2044..027152bfd69 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java @@ -55,14 +55,13 @@ public class ValidateFuzzySearcherTestCase { searcher = new ValidateFuzzySearcher(); } - private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength) { - return "select * from sources * where " + attribute + - " contains ({maxEditDistance:" + maxEditDistance + ", prefixLength:" + prefixLength +"}" + - "fuzzy(\"" + query + "\"))"; + private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength, boolean prefixMatch) { + return "select * from sources * where %s contains ({maxEditDistance:%d,prefixLength:%d,prefix:%b}fuzzy(\"%s\"))" + .formatted(attribute, maxEditDistance, prefixLength, prefixMatch, query); } private String makeQuery(String attribute, String query) { - return makeQuery(attribute, query, 2, 0); + return makeQuery(attribute, query, 2, 0, false); } @@ -76,7 +75,7 @@ public class ValidateFuzzySearcherTestCase { if (validAttributes.contains(attribute)) { assertNull(r.hits().getError()); } else { - assertErrMsg("FUZZY(fuzzy,2,0) " + attribute + ":fuzzy field is not a string attribute", r); + assertErrMsg("FUZZY(fuzzy,2,0,false) " + attribute + ":fuzzy field is not a string attribute", r); } } } @@ -85,28 +84,28 @@ public class ValidateFuzzySearcherTestCase { void testInvalidEmptyStringQuery() { String q = makeQuery("string_single", ""); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(,2,0) string_single: fuzzy query must be non-empty", r); + assertErrMsg("FUZZY(,2,0,false) string_single: fuzzy query must be non-empty", r); } @Test void testInvalidQueryWrongMaxEditDistance() { - String q = makeQuery("string_single", "fuzzy", -1, 0); + String q = makeQuery("string_single", "fuzzy", -1, 0, false); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,-1,0) string_single:fuzzy has invalid maxEditDistance -1: Must be >= 0", r); + assertErrMsg("FUZZY(fuzzy,-1,0,false) string_single:fuzzy has invalid maxEditDistance -1: Must be >= 0", r); } @Test void testInvalidQueryWrongPrefixLength() { - String q = makeQuery("string_single", "fuzzy", 2, -1); + String q = makeQuery("string_single", "fuzzy", 2, -1, true); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,2,-1) string_single:fuzzy has invalid prefixLength -1: Must be >= 0", r); + assertErrMsg("FUZZY(fuzzy,2,-1,true) string_single:fuzzy has invalid prefixLength -1: Must be >= 0", r); } @Test void testInvalidQueryWrongAttributeName() { String q = makeQuery("wrong_name", "fuzzy"); Result r = doSearch(searcher, q); - assertErrMsg("FUZZY(fuzzy,2,0) wrong_name:fuzzy field is not a string attribute", r); + assertErrMsg("FUZZY(fuzzy,2,0,false) wrong_name:fuzzy field is not a string attribute", r); } private static void assertErrMsg(String message, Result r) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java b/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java deleted file mode 100644 index 7835a9934c7..00000000000 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchers.test; - -import com.yahoo.jdisc.Metric; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** -* @author bratseth -*/ -class MockMetric implements Metric { - - private Map<Context, Map<String, Number>> metrics = new HashMap<>(); - - public Map<String, Number> values(Context context) { - return metricsForContext(context); - } - - @Override - public void set(String key, Number val, Context context) { - metricsForContext(context).put(key, val); - } - - @Override - public void add(String key, Number value, Context context) { - Number previousValue = metricsForContext(context).get(key); - if (previousValue == null) - previousValue = 0; - metricsForContext(context).put(key, value.doubleValue() + previousValue.doubleValue()); - } - - /** Returns the metrics for a given context, never null */ - private Map<String, Number> metricsForContext(Context context) { - Map<String, Number> metricsForContext = metrics.get(context); - if (metricsForContext == null) { - metricsForContext = new HashMap<>(); - metrics.put(context, metricsForContext); - } - return metricsForContext; - } - - @Override - public Context createContext(Map<String, ?> dimensions) { - return new MapContext(dimensions); - } - - /** Creates a context containing a single dimension */ - public Metric.Context createContext(String dimensionName, String dimensionValue) { - if (dimensionName.isEmpty()) - return createContext(Collections.emptyMap()); - return createContext(Collections.singletonMap(dimensionName, dimensionValue)); - } - - private class MapContext implements Metric.Context { - - private final Map<String, ?> dimensions; - - public MapContext(Map<String, ?> dimensions) { - this.dimensions = dimensions; - } - - @Override - public int hashCode() { - return dimensions.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ( ! (o instanceof MapContext)) return false; - return dimensions.equals(((MapContext)o).dimensions); - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 87bc602f072..b5e2839c4c0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -27,7 +27,8 @@ import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import java.util.Arrays; + +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -383,7 +384,7 @@ public class VespaSerializerTestCase { private static void newGroupingRequest(Query query, GroupingOperation grouping, Continuation... continuations) { GroupingRequest request = GroupingRequest.newInstance(query); request.setRootOperation(grouping); - request.continuations().addAll(Arrays.asList(continuations)); + request.continuations().addAll(List.of(continuations)); } @Test @@ -463,7 +464,12 @@ public class VespaSerializerTestCase { @Test void testFuzzyAnnotations() { + parseAndConfirm("foo contains ({maxEditDistance:3}fuzzy(\"a\"))"); parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({prefixLength:5,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({maxEditDistance:3,prefix:true}fuzzy(\"a\"))"); + parseAndConfirm("foo contains ({prefix:true}fuzzy(\"a\"))"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 87d18c18db5..675acabe906 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.yql; import static org.junit.jupiter.api.Assertions.*; -import java.util.Arrays; import java.util.List; import com.yahoo.search.schema.DocumentSummary; @@ -48,9 +47,9 @@ public class YqlFieldAndSourceTestCase { DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); mockBackend.addResult(query, result); - mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); - mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, Arrays.asList(FIELD2)); - mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); + mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, List.of(FIELD1, FIELD2)); + mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, List.of(FIELD2)); + mockBackend.addSummaryClassByCopy(THIRD_OPTION, List.of(FIELD3)); searchChain = new Chain<>(new FieldFiller(schemaInfo()), mockBackend); context = Execution.Context.createContextStub(); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 29a651aabf4..91f5984481a 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -437,23 +437,27 @@ public class YqlParserTestCase { QueryTree x = parse("select foo from bar where baz contains fuzzy(\"a b\")"); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("baz", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); - assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, ((FuzzyItem) root).getMaxEditDistance()); - assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, ((FuzzyItem) root).getPrefixLength()); + var fuzzy = (FuzzyItem) root; + assertEquals("baz", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, fuzzy.getMaxEditDistance()); + assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, fuzzy.getPrefixLength()); + assertFalse(fuzzy.isPrefixMatch()); } @Test void testFuzzyAnnotations() { QueryTree x = parse( - "select foo from bar where baz contains ({maxEditDistance: 3, prefixLength: 10}fuzzy(\"a b\"))" + "select foo from bar where baz contains ({maxEditDistance: 3, prefixLength: 10, prefix: true}fuzzy(\"a b\"))" ); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("baz", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); - assertEquals(3, ((FuzzyItem) root).getMaxEditDistance()); - assertEquals(10, ((FuzzyItem) root).getPrefixLength()); + var fuzzy = (FuzzyItem) root; + assertEquals("baz", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(3, fuzzy.getMaxEditDistance()); + assertEquals(10, fuzzy.getPrefixLength()); + assertTrue(fuzzy.isPrefixMatch()); } @Test diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index f4571f04a5d..f863816dab2 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -671,8 +671,39 @@ public class SelectTestCase { QueryTree x = parseWhere("{ \"contains\": [\"description\", { \"fuzzy\": [\"a b\"] }] }"); Item root = x.getRoot(); assertSame(FuzzyItem.class, root.getClass()); - assertEquals("description", ((FuzzyItem) root).getIndexName()); - assertEquals("a b", ((FuzzyItem) root).stringValue()); + var fuzzy = (FuzzyItem) root; + assertEquals("description", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(FuzzyItem.DEFAULT_MAX_EDIT_DISTANCE, fuzzy.getMaxEditDistance()); + assertEquals(FuzzyItem.DEFAULT_PREFIX_LENGTH, fuzzy.getPrefixLength()); + assertFalse(fuzzy.isPrefixMatch()); + } + + @Test + void fuzzy_with_annotations() { + var where = """ + { + "contains": ["description", { + "fuzzy": { + "children": ["a b"], + "attributes": { + "maxEditDistance": 3, + "prefixLength": 10, + "prefix": true + } + } + }] + } + """; + QueryTree x = parseWhere(where); + Item root = x.getRoot(); + assertSame(FuzzyItem.class, root.getClass()); + var fuzzy = (FuzzyItem) root; + assertEquals("description", fuzzy.getIndexName()); + assertEquals("a b", fuzzy.stringValue()); + assertEquals(3, fuzzy.getMaxEditDistance()); + assertEquals(10, fuzzy.getPrefixLength()); + assertTrue(fuzzy.isPrefixMatch()); } //------------------------------------------------------------------- grouping tests diff --git a/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java b/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java index dd2725f2d6e..3fa81e948a2 100644 --- a/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java +++ b/container-search/src/test/java/com/yahoo/text/interpretation/test/AnnotationTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.text.interpretation.test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -50,7 +49,7 @@ public class AnnotationTestCase { } Set<AnnotationClass> annotationClasses = a.getClasses(0, 3); - Set<AnnotationClass> testClass = new HashSet<>(Arrays.asList( + Set<AnnotationClass> testClass = new HashSet<>(List.of( new AnnotationClass("token"), new AnnotationClass("state"))); assertEquals(testClass, annotationClasses); @@ -75,7 +74,7 @@ public class AnnotationTestCase { //This is bad about the API, getTokens may not necessairily return what a user thinks a token is //But it should still be tested a.annotate(0, 1, "n"); - Set<String> testSet = new HashSet<>(Arrays.asList("n", "york", "hotel")); + Set<String> testSet = new HashSet<>(List.of("n", "york", "hotel")); for (Span span :a.getTokens()) { assertTrue(testSet.remove(span.getText())); } @@ -118,7 +117,7 @@ public class AnnotationTestCase { //if a number is not found woe_id = annotations.getInteger("woe_id"); } - assertEquals(Arrays.asList("crab"), toppings); + assertEquals(List.of("crab"), toppings); assertEquals(2459115, woe_id); } diff --git a/container-test/pom.xml b/container-test/pom.xml index 8e1b4870665..d6be6946208 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -61,6 +61,16 @@ <artifactId>onnxruntime</artifactId> </dependency> <dependency> + <groupId>de.kherud</groupId> + <artifactId>llama</artifactId> + <exclusions> + <exclusion> + <groupId>org.jetbrains</groupId> + <artifactId>annotations</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>io.airlift</groupId> <artifactId>airline</artifactId> <exclusions> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 63ac2c306cb..3a06a33ab44 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -205,9 +205,6 @@ function(vespa_use_default_cxx_compiler) if(APPLE) set(DEFAULT_CMAKE_C_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/gcc-13") set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/g++-13") - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") - set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 22.04" AND EXISTS "/usr/bin/gcc-12" AND EXISTS "/usr/bin/g++-12") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc-12") diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 071ef066e93..e34ccddf8e3 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -68,8 +68,8 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.696</aws-sdk.vespa.version> - <athenz.vespa.version>1.11.55</athenz.vespa.version> + <aws-sdk.vespa.version>1.12.706</aws-sdk.vespa.version> + <athenz.vespa.version>1.11.56</athenz.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -78,7 +78,7 @@ find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> - <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> + <bouncycastle.vespa.version>1.78.1</bouncycastle.vespa.version> <byte-buddy.vespa.version>1.14.13</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> @@ -91,7 +91,7 @@ <commons-logging.vespa.version>1.3.1</commons-logging.vespa.version> <!-- Bindings exported by jdisc through jcl-over-slf4j. --> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> <commons-compress.vespa.version>1.26.1</commons-compress.vespa.version> - <commons-cli.vespa.version>1.6.0</commons-cli.vespa.version> + <commons-cli.vespa.version>1.7.0</commons-cli.vespa.version> <curator.vespa.version>5.6.0</curator.vespa.version> <dropwizard.metrics.vespa.version>4.2.25</dropwizard.metrics.vespa.version> <!-- ZK 3.9.1 requires this --> <eclipse-angus.vespa.version>2.0.2</eclipse-angus.vespa.version> @@ -104,7 +104,7 @@ <hamcrest.vespa.version>2.2</hamcrest.vespa.version> <hdrhistogram.vespa.version>2.1.12</hdrhistogram.vespa.version> <huggingface.vespa.version>0.27.0</huggingface.vespa.version> - <icu4j.vespa.version>74.2</icu4j.vespa.version> + <icu4j.vespa.version>75.1</icu4j.vespa.version> <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> <java-jwt.vespa.version>4.4.0</java-jwt.vespa.version> <javax.annotation.vespa.version>1.2</javax.annotation.vespa.version> @@ -126,7 +126,7 @@ <mimepull.vespa.version>1.10.0</mimepull.vespa.version> <mockito.vespa.version>5.11.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> - <netty.vespa.version>4.1.108.Final</netty.vespa.version> + <netty.vespa.version>4.1.109.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.65.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.17.1</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.2</opennlp.vespa.version> @@ -137,10 +137,10 @@ <plexus-archiver.vespa.version>4.9.2</plexus-archiver.vespa.version> <plexus-interpolation.vespa.version>1.27</plexus-interpolation.vespa.version> <plexus-io.vespa.version>3.4.2</plexus-io.vespa.version> - <plexus-utils.vespa.version>4.0.0</plexus-utils.vespa.version> + <plexus-utils.vespa.version>4.0.1</plexus-utils.vespa.version> <plexus-xml.vespa.version>4.0.3</plexus-xml.vespa.version> <protobuf.vespa.version>3.25.3</protobuf.vespa.version> - <questdb.vespa.version>7.3.10</questdb.vespa.version> + <questdb.vespa.version>7.4.2</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> <surefire.vespa.version>3.2.5</surefire.vespa.version> @@ -174,14 +174,14 @@ <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> <maven-failsafe-plugin.vespa.version>3.2.5</maven-failsafe-plugin.vespa.version> - <maven-gpg-plugin.vespa.version>3.2.2</maven-gpg-plugin.vespa.version> + <maven-gpg-plugin.vespa.version>3.2.4</maven-gpg-plugin.vespa.version> <maven-install-plugin.vespa.version>3.1.1</maven-install-plugin.vespa.version> - <maven-jar-plugin.vespa.version>3.3.0</maven-jar-plugin.vespa.version> + <maven-jar-plugin.vespa.version>3.4.1</maven-jar-plugin.vespa.version> <maven-javadoc-plugin.vespa.version>3.6.3</maven-javadoc-plugin.vespa.version> <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> <maven-plugin-tools.vespa.version>3.12.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> - <maven-resolver.vespa.version>1.9.18</maven-resolver.vespa.version> + <maven-resolver.vespa.version>1.9.19</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.2</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.1</maven-source-plugin.vespa.version> diff --git a/dist/vespa.spec b/dist/vespa.spec index 2c90b99a9dc..33c969ce1a6 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -205,6 +205,9 @@ Requires: vespa-protobuf = %{_vespa_protobuf_version} Requires: llvm-libs %endif Requires: vespa-onnxruntime = 1.17.1 +%if 0%{?el8} || 0%{?el9} +Requires: vespa-jllama = 3.0.0 +%endif %description libs diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index cc9d7b9e8b7..c8c536cb4c1 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -9,7 +9,6 @@ import com.yahoo.document.DocumentOperation; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -221,7 +220,7 @@ public final class Processing extends ProcessingAccess { @Override protected List<DocumentOperation> getOnceOperationsToBeProcessed() { if (operationsGotten) - return Collections.emptyList(); + return List.of(); operationsGotten = true; return getDocumentOperations(); diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java index aacac7b5c04..822359d7c09 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java @@ -25,9 +25,8 @@ import com.yahoo.messagebus.Reply; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; @@ -48,7 +47,7 @@ public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentPr this.type.addField(new Field("blahblah", DataType.STRING)); this.type.addField(new Field("defaultWait", DataType.INT)); this.type.addField(new Field("customWait", DataType.INT)); - type.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("blahblah", "defaultWait", "customWait"))); + type.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("blahblah", "defaultWait", "customWait"))); } @Test @@ -158,8 +157,7 @@ public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentPr Document doc = ((DocumentPut) op).getDocument(); for (Field f : doc.getDataType().fieldSet()) { FieldValue val = doc.getFieldValue(f); - if (val instanceof StringFieldValue) { - StringFieldValue sf = (StringFieldValue) val; + if (val instanceof StringFieldValue sf) { doc.setFieldValue(f, new StringFieldValue(sf.getString().toUpperCase())); } } diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java index 2f07958e39e..7abfdc324b3 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java @@ -21,7 +21,6 @@ import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; -import com.yahoo.document.update.MapValueUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory; @@ -33,8 +32,7 @@ import com.yahoo.vespa.indexinglanguage.parser.ParseException; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -160,7 +158,7 @@ public class DocumentScriptTestCase { assertSpanTrees(str, "mySpanTree"); } - private class FieldPathFixture { + private static class FieldPathFixture { final DocumentType type; final StructDataType structType; final DataType structMap; @@ -264,7 +262,7 @@ public class DocumentScriptTestCase { } private static DocumentScript newScript(DocumentType docType, String fieldName) { - return new DocumentScript(docType.getName(), Collections.singletonList(fieldName), + return new DocumentScript(docType.getName(), List.of(fieldName), new StatementExpression(new InputExpression(fieldName), new IndexExpression(fieldName))); } @@ -285,7 +283,7 @@ public class DocumentScriptTestCase { private static void assertSpanTrees(FieldValue actual, String... expectedSpanTrees) { assertTrue(actual instanceof StringFieldValue); StringFieldValue str = (StringFieldValue)actual; - assertEquals(new ArrayList<>(Arrays.asList(expectedSpanTrees)), + assertEquals(List.of(expectedSpanTrees), new ArrayList<>(str.getSpanTreeMap().keySet())); } @@ -359,7 +357,7 @@ public class DocumentScriptTestCase { } private static DocumentScript newScript() throws ParseException { - return new DocumentScript("documentType", Arrays.asList("documentField"), + return new DocumentScript("documentType", List.of("documentField"), Expression.fromString("input documentField | index documentField")); } } diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 38bd3bfdeca..3b58b50dc4f 100644 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -62,13 +62,13 @@ public class DocumentType extends StructuredDataType { * @param contentStructType The type of the content struct */ public DocumentType(String name, StructDataType contentStructType) { - this(name, contentStructType, Collections.emptySet()); + this(name, contentStructType, Set.of()); } public DocumentType(String name, StructDataType contentStructType, Set<String> importedFieldNames) { super(name); this.contentStructType = contentStructType; - this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); + this.importedFieldNames = Set.copyOf(importedFieldNames); } public DocumentType(String name, Set<String> importedFieldNames) { diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java index b78e48d2b41..7596e359d99 100755..100644 --- a/document/src/main/java/com/yahoo/document/FieldPath.java +++ b/document/src/main/java/com/yahoo/document/FieldPath.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -19,7 +18,7 @@ public class FieldPath implements Iterable<FieldPathEntry> { * Constructs an empty path. */ public FieldPath() { - list = Collections.emptyList(); + list = List.of(); } /** @@ -109,7 +108,7 @@ public class FieldPath implements Iterable<FieldPathEntry> { FieldPathEntry.Type type = entry.getType(); switch (type) { case STRUCT_FIELD: - if (out.length() > 0) { + if (!out.isEmpty()) { out.append("."); } Field field = entry.getFieldRef(); diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java index 4750e64a784..312f2800e6c 100644 --- a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java @@ -3,7 +3,6 @@ package com.yahoo.document.annotation; import com.yahoo.document.DataType; -import java.util.Arrays; import java.util.List; /** @@ -29,7 +28,7 @@ public final class AnnotationTypes { public static final AnnotationType PROXIMITY_BREAK = new AnnotationType("proximity_break", DataType.DOUBLE, 8); public static final AnnotationType SPECIAL_TOKEN = new AnnotationType("special_token", 9); - public static final List<AnnotationType> ALL_TYPES = Arrays.asList(TERM, TOKEN_TYPE, CANONICAL, NORMALIZED, READING, + public static final List<AnnotationType> ALL_TYPES = List.of(TERM, TOKEN_TYPE, CANONICAL, NORMALIZED, READING, STEM, TRANSFORMED, PROXIMITY_BREAK, SPECIAL_TOKEN); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java index 35d12f32906..42405bb6f06 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java @@ -13,6 +13,7 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.serialization.DocumentUpdateReader; import com.yahoo.document.serialization.VespaDocumentSerializer6; import java.util.ListIterator; +import java.util.Objects; /** * @author Thomas Gundersen @@ -48,8 +49,8 @@ public abstract class FieldPathUpdate { private DocumentSelector selector; private String originalFieldPath; private String whereClause; - private Type updType; - private DocumentType docType; + private final Type updType; + private final DocumentType docType; public FieldPathUpdate(Type updType, DocumentType docType, String fieldPath, String whereClause) { this.updType = updType; @@ -97,7 +98,7 @@ public abstract class FieldPathUpdate { public void setWhereClause(String whereClause) throws ParseException { this.whereClause = whereClause; selector = null; - if (whereClause != null && !whereClause.equals("")) { + if (whereClause != null && !whereClause.isEmpty()) { selector = new DocumentSelector(whereClause); } } @@ -135,15 +136,11 @@ public abstract class FieldPathUpdate { } public static FieldPathUpdate create(Type type, DocumentType docType, DocumentUpdateReader reader) { - switch (type) { - case ASSIGN: - return new AssignFieldPathUpdate(docType, reader); - case ADD: - return new AddFieldPathUpdate(docType, reader); - case REMOVE: - return new RemoveFieldPathUpdate(docType, reader); - } - throw new IllegalArgumentException("Field path update type '" + type + "' not supported."); + return switch (type) { + case ASSIGN -> new AssignFieldPathUpdate(docType, reader); + case ADD -> new AddFieldPathUpdate(docType, reader); + case REMOVE -> new RemoveFieldPathUpdate(docType, reader); + }; } @Override @@ -153,10 +150,10 @@ public abstract class FieldPathUpdate { FieldPathUpdate that = (FieldPathUpdate) o; - if (docType != null ? !docType.equals(that.docType) : that.docType != null) return false; - if (originalFieldPath != null ? !originalFieldPath.equals(that.originalFieldPath) : that.originalFieldPath != null) + if (!Objects.equals(docType, that.docType)) return false; + if (!Objects.equals(originalFieldPath, that.originalFieldPath)) return false; - if (whereClause != null ? !whereClause.equals(that.whereClause) : that.whereClause != null) return false; + if (!Objects.equals(whereClause, that.whereClause)) return false; return true; } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index b6cf8c6e18b..358c0cb65e4 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -30,7 +30,7 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart * valid JSON representation of a feed. * * @author Steinar Knutsen - * @author dybis + * @author Haakon Dybdahl */ public class JsonReader { diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java index 656fb8dd033..334a994ebe8 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java @@ -1,7 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespaxmlparser; -import com.yahoo.document.*; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.MapDataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java index 5f8e95a2b4f..17d1d745f6a 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java @@ -3,15 +3,14 @@ package com.yahoo.document; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; /** * @author Thomas Gundersen @@ -37,7 +36,7 @@ public class DocumentTypeTestCase { root.addField("rootfield1", DataType.STRING); root.addField("rootfield2", DataType.STRING); root.addField("rootfield3", DataType.STRING); - root.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("rootfield2"))); + root.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("rootfield2"))); assertEquals(1, root.fieldSet().size()); assertEquals("rootfield2", root.fieldSet().iterator().next().getName()); assertEquals(3, root.fieldSetAll().size()); @@ -49,25 +48,25 @@ public class DocumentTypeTestCase { DocumentType root = new DocumentType("root"); root.addField("rootfield", DataType.STRING); - root.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("rootfield"))); + root.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("rootfield"))); DocumentType parent1 = new DocumentType("parent1"); parent1.addField("overridden", DataType.STRING); parent1.addField("parent1field", DataType.STRING); parent1.inherit(root); - parent1.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("parent1field", "overridden"))); + parent1.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("parent1field", "overridden"))); DocumentType parent2 = new DocumentType("parent2"); parent2.addField("parent2field", DataType.STRING); parent2.inherit(root); - parent2.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("parent2field"))); + parent2.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("parent2field"))); DocumentType child = new DocumentType("child"); child.addField("childfield", DataType.INT); child.addField("overridden", DataType.STRING); child.inherit(parent1); child.inherit(parent2); - child.addFieldSets(Collections.singletonMap(DocumentType.DOCUMENT, Arrays.asList("childfield", "overridden"))); + child.addFieldSets(Map.of(DocumentType.DOCUMENT, List.of("childfield", "overridden"))); typeManager.register(root); typeManager.register(parent1); @@ -77,19 +76,19 @@ public class DocumentTypeTestCase { Iterator inherited = child.getInheritedTypes().iterator(); assertEquals(parent1, inherited.next()); assertEquals(parent2, inherited.next()); - assertTrue(!inherited.hasNext()); + assertFalse(inherited.hasNext()); inherited = parent1.getInheritedTypes().iterator(); assertEquals(root, inherited.next()); - assertTrue(!inherited.hasNext()); + assertFalse(inherited.hasNext()); inherited = parent2.getInheritedTypes().iterator(); assertEquals(root, inherited.next()); - assertTrue(!inherited.hasNext()); + assertFalse(inherited.hasNext()); inherited = root.getInheritedTypes().iterator(); assertEquals(DataType.DOCUMENT, inherited.next()); - assertTrue(!inherited.hasNext()); + assertFalse(inherited.hasNext()); Iterator fields = child.fieldSet().iterator(); Field field; diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index aa043a25d78..e72d3720024 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -20,7 +20,6 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; -import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; @@ -32,7 +31,6 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.internal.GeoPosType; import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; @@ -64,7 +62,6 @@ import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 63255f90919..b09d33a2fe4 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -32,7 +32,6 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -852,10 +851,10 @@ public class DocumentSelectorTestCase { @Test public void testThatSelectionIsConvertedToQueries() throws ParseException { - assertThatQueriesAreCreated("music.expire > now()", Arrays.asList("music"), Arrays.asList("expire:>now(0)")); - assertThatQueriesAreCreated("music.expire > now() - 300", Arrays.asList("music"), Arrays.asList("expire:>now(300)")); - assertThatQueriesAreCreated("music.expire > now() - 300 and video.expire > now() - 3600", Arrays.asList("music", "video"), Arrays.asList("expire:>now(300)", "expire:>now(3600)")); - assertThatQueriesAreCreated("music.expire > now() - 300 or video", Arrays.asList("music"), Arrays.asList("expire:>now(300)")); + assertThatQueriesAreCreated("music.expire > now()", List.of("music"), List.of("expire:>now(0)")); + assertThatQueriesAreCreated("music.expire > now() - 300", List.of("music"), List.of("expire:>now(300)")); + assertThatQueriesAreCreated("music.expire > now() - 300 and video.expire > now() - 3600", List.of("music", "video"), List.of("expire:>now(300)", "expire:>now(3600)")); + assertThatQueriesAreCreated("music.expire > now() - 300 or video", List.of("music"), List.of("expire:>now(300)")); assertVisitWithInvalidNowFails("music.field1 > now() - 300 and music.field2 > now() - 300", "Specifying multiple document types is not allowed"); assertVisitWithInvalidNowFails("music.field1 > now() - 300 and video.field1 > now() and music.field2 > now() - 300", "Specifying multiple document types is not allowed"); assertVisitWithInvalidNowFails("now() > music.field", "Left hand side of comparison must be a document field"); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 68a8f964575..21771558f05 100755..100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -18,6 +18,8 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; + +import java.util.List; import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; @@ -37,7 +39,6 @@ import com.yahoo.messagebus.routing.RoutingTable; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vdslib.state.ClusterState; -import java.util.Arrays; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -162,14 +163,13 @@ public class MessageBusVisitorSession implements VisitorSession { } VisitorControlHandler.CompletionCode toCompletionCode() { - switch (state) { - case COMPLETED: return VisitorControlHandler.CompletionCode.SUCCESS; - case ABORTED: return VisitorControlHandler.CompletionCode.ABORTED; - case FAILED: return VisitorControlHandler.CompletionCode.FAILURE; - case TIMED_OUT: return VisitorControlHandler.CompletionCode.TIMEOUT; - default: - throw new IllegalStateException("Current state did not have a valid value: " + state); - } + return switch (state) { + case COMPLETED -> VisitorControlHandler.CompletionCode.SUCCESS; + case ABORTED -> VisitorControlHandler.CompletionCode.ABORTED; + case FAILED -> VisitorControlHandler.CompletionCode.FAILURE; + case TIMED_OUT -> VisitorControlHandler.CompletionCode.TIMEOUT; + default -> throw new IllegalStateException("Current state did not have a valid value: " + state); + }; } public boolean failed() { @@ -312,9 +312,7 @@ public class MessageBusVisitorSession implements VisitorSession { return sessionCounter.incrementAndGet(); } private static String createSessionName() { - StringBuilder sb = new StringBuilder(); - sb.append("visitor-").append(getNextSessionId()).append('-').append(System.currentTimeMillis()); - return sb.toString(); + return "visitor-" + getNextSessionId() + '-' + System.currentTimeMillis(); } private final VisitorParameters params; @@ -647,7 +645,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.getTrace().setLevel(params.getTraceLevel()); msg.setTimeRemaining(messageTimeoutMs); - msg.setBuckets(Arrays.asList(bucket.getSuperbucket(), bucket.getProgress())); + msg.setBuckets(List.of(bucket.getSuperbucket(), bucket.getProgress())); msg.setDocumentSelection(params.getDocumentSelection()); msg.setBucketSpace(params.getBucketSpace()); msg.setFromTimestamp(params.getFromTimestamp()); @@ -881,7 +879,7 @@ public class MessageBusVisitorSession implements VisitorSession { } try { - if (msg.getErrorMessage().length() > 0) { + if (!msg.getErrorMessage().isEmpty()) { params.getControlHandler().onVisitorError(msg.getErrorMessage()); } synchronized (progress.getToken()) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 0a0fc5f4991..5eb5b6f2567 100755..100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -16,7 +16,6 @@ import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; import com.yahoo.vespa.config.content.DistributionConfig; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -283,7 +282,7 @@ public class DocumentProtocol implements Protocol { // Prepare version specifications to use when adding routable factories. VersionSpecification version6 = new VersionSpecification(6, 221); - List<VersionSpecification> from6 = Collections.singletonList(version6); + List<VersionSpecification> from6 = List.of(version6); // 6.x serialization (keep alphabetized please) putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6); @@ -322,7 +321,7 @@ public class DocumentProtocol implements Protocol { private void registerV8Factories() { var version8 = new VersionSpecification(8, 310); // Must be same as in C++ impl - var from8 = Collections.singletonList(version8); + var from8 = List.of(version8); putRoutableFactory(MESSAGE_CREATEVISITOR, RoutableFactories80.createCreateVisitorMessageFactory(), from8); putRoutableFactory(MESSAGE_DESTROYVISITOR, RoutableFactories80.createDestroyVisitorMessageFactory(), from8); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java index 4ca6fb03821..77ec8f175df 100755..100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java @@ -3,7 +3,6 @@ package com.yahoo.documentapi.messagebus.systemstate.rule; import java.util.ArrayList; import java.util.List; -import java.util.Arrays; /** * @author Simon Thoresen Hult @@ -25,7 +24,7 @@ public class Location { * @param loc The location string to parse. */ public Location(String loc) { - items.addAll(Arrays.asList(loc.split("/"))); + items.addAll(List.of(loc.split("/"))); normalize(); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java index 0bbd3549869..a9d6a7a4cbf 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.jrt.slobrok.api.Mirror; import org.junit.Test; -import java.util.Arrays; import java.util.List; import java.util.Random; @@ -55,7 +54,7 @@ public class LoadBalancerTestCase { public void testAdaptiveLoadBalancer() { LoadBalancer lb = new AdaptiveLoadBalancer("foo", new Random(1)); - List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), + List<Mirror.Entry> entries = List.of(new Mirror.Entry("foo/0/default", "tcp/bar:1"), new Mirror.Entry("foo/1/default", "tcp/bar:2"), new Mirror.Entry("foo/2/default", "tcp/bar:3")); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); @@ -112,7 +111,7 @@ public class LoadBalancerTestCase { private void verifyLoadBalancerOneItemOnly(LoadBalancer lb) { - List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); + List<Mirror.Entry> entries = List.of(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index 25bdeb60013..472ce00aac3 100755..100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -52,7 +52,6 @@ import org.junit.Ignore; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -116,12 +115,12 @@ public class PolicyTestCase { frame.setHop(new HopSpec("test", "[AND]") .addRecipient("foo") .addRecipient("bar")); - frame.assertSelect(Arrays.asList("foo", "bar")); + frame.assertSelect(List.of("foo", "bar")); frame.setHop(new HopSpec("test", "[AND:baz]") .addRecipient("foo") .addRecipient("bar")); - frame.assertSelect(Arrays.asList("baz")); // param precedes recipients + frame.assertSelect(List.of("baz")); // param precedes recipients frame.setHop(new HopSpec("test", "[AND:foo]")); frame.assertMergeOneReply("foo"); @@ -414,10 +413,10 @@ public class PolicyTestCase { PolicyTestFrame frame = createFrameWithTwoRoutes(); frame.setMessage(createRemove("id:ns:testdoc::1")); - frame.assertSelect(Arrays.asList("testdoc-route")); + frame.assertSelect(List.of("testdoc-route")); frame.setMessage(createRemove("id:ns:other::1")); - frame.assertSelect(Arrays.asList("other-route")); + frame.assertSelect(List.of("other-route")); frame.destroy(); } @@ -426,10 +425,10 @@ public class PolicyTestCase { PolicyTestFrame frame = createFrameWithTwoRoutes(); frame.setMessage(createGet("id:ns:testdoc::1")); - frame.assertSelect(Arrays.asList("testdoc-route")); + frame.assertSelect(List.of("testdoc-route")); frame.setMessage(createGet("id:ns:other::1")); - frame.assertSelect(Arrays.asList("other-route")); + frame.assertSelect(List.of("other-route")); frame.destroy(); } @@ -586,19 +585,19 @@ public class PolicyTestCase { "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar")); frame.setMessage(new GetDocumentMessage(new DocumentId("id:ns:testdoc::"), "fieldSet")); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), new DocumentId("id:ns:testdoc::")))); frame.setMessage(put); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); frame.setMessage(new RemoveDocumentMessage(new DocumentId("id:ns:testdoc::"))); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"), new DocumentId("id:ns:testdoc::")))); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); frame.setMessage(put); frame.assertMergeOneReply("foo"); @@ -619,13 +618,13 @@ public class PolicyTestCase { "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar")); frame.setMessage(new GetDocumentMessage(new DocumentId("id:ns:testdoc::"), "fieldSet")); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); Document doc = new Document(manager.getDocumentType("testdoc"), new DocumentId("id:ns:testdoc::")); doc.setFieldValue("intfield", 3000); Message put = new PutDocumentMessage(new DocumentPut(doc)); frame.setMessage(put); - frame.assertSelect(Arrays.asList("foo")); + frame.assertSelect(List.of("foo")); frame.setMessage(put); frame.assertMergeOneReply("foo"); @@ -653,7 +652,7 @@ public class PolicyTestCase { frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"), new DocumentId("id:ns:testdoc::")))); - frame.assertSelect(Arrays.asList("docproc/cluster.foo")); + frame.assertSelect(List.of("docproc/cluster.foo")); frame.destroy(); } @@ -667,7 +666,7 @@ public class PolicyTestCase { assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1)); frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]")); - assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default")); + assertSelect(frame, 1, List.of(frame.getNetwork().getConnectionSpec() + "/chain.default")); frame.destroy(); } @@ -685,16 +684,16 @@ public class PolicyTestCase { .addRecipient("docproc/cluster.default/3/chain.default") .addRecipient("docproc/cluster.default/6/chain.default") .addRecipient("docproc/cluster.default/9/chain.default")); - assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default", + assertSelect(frame, 32, List.of("docproc/cluster.default/3/chain.default", "docproc/cluster.default/6/chain.default", "docproc/cluster.default/9/chain.default")); frame.getNetwork().unregisterSession("6/chain.default"); assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9)); - assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default", + assertSelect(frame, 32, List.of("docproc/cluster.default/3/chain.default", "docproc/cluster.default/9/chain.default")); frame.getNetwork().unregisterSession("3/chain.default"); assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8)); - assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default")); + assertSelect(frame, 32, List.of("docproc/cluster.default/9/chain.default")); frame.getNetwork().unregisterSession("9/chain.default"); assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7)); assertSelect(frame, 32, new ArrayList<>()); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java index f51f285723f..8e2fcdcd946 100755..100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -28,7 +28,6 @@ import com.yahoo.messagebus.test.SimpleProtocol; import com.yahoo.messagebus.test.SimpleReply; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -219,7 +218,7 @@ public class PolicyTestFrame { * @param recipient The expected recipient. */ public void assertMergeOneReply(String recipient) { - assertSelect(Arrays.asList(recipient)); + assertSelect(List.of(recipient)); Map<String, Integer> replies = new HashMap<>(); replies.put(recipient, ErrorCode.NONE); @@ -237,7 +236,7 @@ public class PolicyTestFrame { * @param recipientTwo The second expected recipient. */ public void assertMergeTwoReplies(String recipientOne, String recipientTwo) { - assertSelect(Arrays.asList(recipientOne, recipientTwo)); + assertSelect(List.of(recipientOne, recipientTwo)); Map<String, Integer> replies = new HashMap<>(); replies.put(recipientOne, ErrorCode.NONE); @@ -254,7 +253,7 @@ public class PolicyTestFrame { replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR); replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR); - assertMerge(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR), List.of(recipientOne, recipientTwo)); + assertMerge(replies, List.of(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR), List.of(recipientOne, recipientTwo)); replies.put(recipientOne, ErrorCode.NONE); replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java index 681c47209cb..bca2922ab2c 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java @@ -8,7 +8,6 @@ import org.junit.Test; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -26,7 +25,7 @@ public class PriorityTestCase { String path = "test/crosslanguagefiles/5.1-Priority.txt"; BufferedReader in = new BufferedReader(new FileReader(path)); - List<Priority> expected = new LinkedList<Priority>(Arrays.asList(Priority.values())); + List<Priority> expected = new LinkedList<Priority>(List.of(Priority.values())); String str; while ((str = in.readLine()) != null) { String arr[] = str.split(":", 2); diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index cae11d66d13..625906fc6a2 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -66,7 +66,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -538,7 +537,7 @@ public class DocumentGenPluginTest { } private static DocumentTypeManager typeManagerFromSDs(String... files) { - var cfg = getDocumentConfig(Arrays.asList(files)); + var cfg = getDocumentConfig(List.of(files)); return new DocumentTypeManager(cfg); } diff --git a/eval/src/vespa/eval/eval/cell_type.h b/eval/src/vespa/eval/eval/cell_type.h index c15a5b68dba..b7063f84a96 100644 --- a/eval/src/vespa/eval/eval/cell_type.h +++ b/eval/src/vespa/eval/eval/cell_type.h @@ -6,8 +6,8 @@ #include <vespa/vespalib/util/bfloat16.h> #include <vespa/vespalib/util/typify.h> #include <vector> -#include <cstdint> #include <cassert> +#include <cstdlib> namespace vespalib::eval { @@ -70,7 +70,7 @@ struct CellMetaNotScalar { struct CellMeta { const CellType cell_type; const bool is_scalar; - constexpr CellMeta(CellType cell_type_in, bool is_scalar_in) + constexpr CellMeta(CellType cell_type_in, bool is_scalar_in) noexcept : cell_type(cell_type_in), is_scalar(is_scalar_in) { // is_scalar -> double cell type @@ -105,9 +105,9 @@ struct CellMeta { // normalize to make sure scalar values have cell type double static constexpr CellMeta normalize(CellType cell_type, bool is_scalar) { if (is_scalar) { - return CellMeta(CellType::DOUBLE, true); + return {CellType::DOUBLE, true}; } else { - return CellMeta(cell_type, false); + return {cell_type, false}; } } diff --git a/eval/src/vespa/eval/eval/int8float.cpp b/eval/src/vespa/eval/eval/int8float.cpp index 14a8d45c4cd..5e3fab7b615 100644 --- a/eval/src/vespa/eval/eval/int8float.cpp +++ b/eval/src/vespa/eval/eval/int8float.cpp @@ -1,3 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "int8float.h" +#include <vespa/vespalib/objects/nbostream.h> + + +namespace vespalib::eval { + +nbostream &operator<<(nbostream &stream, Int8Float v) { + return stream << v.get_bits(); +} + +nbostream &operator>>(nbostream &stream, Int8Float &v) { + int8_t byte; + stream >> byte; + v.assign_bits(byte); + return stream; +} + +} diff --git a/eval/src/vespa/eval/eval/int8float.h b/eval/src/vespa/eval/eval/int8float.h index a503f19fe55..b751b2eb8ad 100644 --- a/eval/src/vespa/eval/eval/int8float.h +++ b/eval/src/vespa/eval/eval/int8float.h @@ -3,7 +3,8 @@ #pragma once #include <cstdint> -#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib { class nbostream; } namespace vespalib::eval { @@ -35,12 +36,7 @@ public: constexpr void assign_bits(int8_t value) noexcept { _bits = value; } }; -inline nbostream & operator << (nbostream &stream, Int8Float v) { return stream << v.get_bits(); } -inline nbostream & operator >> (nbostream &stream, Int8Float & v) { - int8_t byte; - stream >> byte; - v.assign_bits(byte); - return stream; -} +nbostream & operator << (nbostream &stream, Int8Float v); +nbostream & operator >> (nbostream &stream, Int8Float & v); } diff --git a/eval/src/vespa/eval/eval/typed_cells.h b/eval/src/vespa/eval/eval/typed_cells.h index d05c3e3294a..3dd8c30a3a9 100644 --- a/eval/src/vespa/eval/eval/typed_cells.h +++ b/eval/src/vespa/eval/eval/typed_cells.h @@ -11,24 +11,24 @@ namespace vespalib::eval { struct TypedCells { const void *data; - size_t size:56; - CellType type; + size_t size:56; + CellType type; - explicit TypedCells(ConstArrayRef<double> cells) : data(cells.begin()), size(cells.size()), type(CellType::DOUBLE) {} - explicit TypedCells(ConstArrayRef<float> cells) : data(cells.begin()), size(cells.size()), type(CellType::FLOAT) {} - explicit TypedCells(ConstArrayRef<BFloat16> cells) : data(cells.begin()), size(cells.size()), type(CellType::BFLOAT16) {} - explicit TypedCells(ConstArrayRef<Int8Float> cells) : data(cells.begin()), size(cells.size()), type(CellType::INT8) {} + explicit TypedCells(ConstArrayRef<double> cells) noexcept : data(cells.begin()), size(cells.size()), type(CellType::DOUBLE) {} + explicit TypedCells(ConstArrayRef<float> cells) noexcept : data(cells.begin()), size(cells.size()), type(CellType::FLOAT) {} + explicit TypedCells(ConstArrayRef<BFloat16> cells) noexcept : data(cells.begin()), size(cells.size()), type(CellType::BFLOAT16) {} + explicit TypedCells(ConstArrayRef<Int8Float> cells) noexcept : data(cells.begin()), size(cells.size()), type(CellType::INT8) {} TypedCells() noexcept : data(nullptr), size(0), type(CellType::DOUBLE) {} TypedCells(const void *dp, CellType ct, size_t sz) noexcept : data(dp), size(sz), type(ct) {} - template <typename T> bool check_type() const { return vespalib::eval::check_cell_type<T>(type); } + template <typename T> bool check_type() const noexcept { return check_cell_type<T>(type); } - template <typename T> ConstArrayRef<T> typify() const { + template <typename T> ConstArrayRef<T> typify() const noexcept { assert(check_type<T>()); return ConstArrayRef<T>((const T *)data, size); } - template <typename T> ConstArrayRef<T> unsafe_typify() const { + template <typename T> ConstArrayRef<T> unsafe_typify() const noexcept { return ConstArrayRef<T>((const T *)data, size); } @@ -36,6 +36,7 @@ struct TypedCells { TypedCells(const TypedCells &other) noexcept = default; TypedCells & operator= (TypedCells &&other) noexcept = default; TypedCells & operator= (const TypedCells &other) noexcept = default; + bool valid() const noexcept { return size != 0; } }; } // namespace diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h index 235e7ebb4b4..0f79f94ee7d 100644 --- a/eval/src/vespa/eval/eval/value.h +++ b/eval/src/vespa/eval/eval/value.h @@ -6,6 +6,7 @@ #include "value_type.h" #include "typed_cells.h" #include <vespa/vespalib/util/string_id.h> +#include <memory> namespace vespalib::eval { @@ -16,7 +17,7 @@ struct Value { using UP = std::unique_ptr<Value>; using CREF = std::reference_wrapper<const Value>; virtual const ValueType &type() const = 0; - virtual ~Value() {} + virtual ~Value() = default; // Root lookup structure for mapping labels to dense subspace indexes struct Index { @@ -40,7 +41,7 @@ struct Value { // create_view will be extracted here. virtual bool next_result(ConstArrayRef<string_id*> addr_out, size_t &idx_out) = 0; - virtual ~View() {} + virtual ~View() = default; }; // total number of mappings (equal to the number of dense subspaces) @@ -50,7 +51,7 @@ struct Value { // labels from a subset of the mapped dimensions. virtual std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const = 0; - virtual ~Index() {} + virtual ~Index() = default; }; virtual TypedCells cells() const = 0; virtual const Index &index() const = 0; diff --git a/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp index 81f25241d3d..94f0a313f2e 100644 --- a/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp +++ b/eval/src/vespa/eval/instruction/dense_hamming_distance.cpp @@ -3,7 +3,6 @@ #include "dense_hamming_distance.h" #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/hamming_distance.h> #include <vespa/vespalib/util/binary_hamming_distance.h> #include <vespa/log/log.h> diff --git a/fbench/src/test/filereader.cpp b/fbench/src/test/filereader.cpp index 87c5914e85b..b2061633d41 100644 --- a/fbench/src/test/filereader.cpp +++ b/fbench/src/test/filereader.cpp @@ -66,7 +66,7 @@ main(int argc, char **argv) return -1; } int res; - int buflen = 10240; + constexpr int buflen = 10240; char buf[buflen]; while ((res = reader->ReadLine(buf, buflen - 1)) >= 0) { // printf("len=%d, content:>%s<\n", res, buf); diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 2442ffa7b1f..0007c4e0127 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -23,7 +23,7 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -148,7 +148,7 @@ public class FileDownloaderTest { File barFile = new File(subdir, "really-long-filename-over-100-bytes-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); IOUtils.writeFile(barFile, "bar", false); - File tarFile = new FileReferenceCompressor(compressed, gzip).compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); + File tarFile = new FileReferenceCompressor(compressed, gzip).compress(tempPath.toFile(), List.of(fooFile, barFile), new File(tempPath.toFile(), filename)); byte[] tarredContent = IOUtils.readFileBytes(tarFile); receiveFile(fileReference, filename, compressed, tarredContent); Optional<File> downloadedFile = getFile(fileReference); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java b/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java index 328d581aed3..d6affae5b03 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Dimension.java @@ -44,8 +44,8 @@ public enum Dimension { /** * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. * - * <p><em>Eager resolution</em>: This dimension is resolved before putting the flag data to the config server - * or controller, unless controller and the flag has declared this dimension. + * <p><em>Eager resolution</em>: This dimension is resolved before storing the flag data in the config server and + * controller ZooKeeper, UNLESS it is the controller and the flag was defined with this dimension in [Permanent]Flags. */ CLOUD("cloud"), @@ -61,10 +61,19 @@ public enum Dimension { /** Email address of user - provided by auth0 in console. */ CONSOLE_USER_EMAIL("console-user-email"), - /** Hosted Vespa environment from com.yahoo.config.provision.Environment::value, e.g. prod, staging, test. */ + /** + * Hosted Vespa environment from com.yahoo.config.provision.Environment::value, e.g. prod, staging, test. + * <em>Eager resolution</em>, see {@link #CLOUD}. + */ ENVIRONMENT("environment"), /** + * The machine flavor from com.yahoo.vespa.hosted.spec.VespaFlavor::vespaFlavorName, e.g. aws-g4dn.xlarge, + * gcp-n2d-highmem-2-375, C-2E/64/960. + */ + FLAVOR("flavor"), + + /** * Fully qualified hostname. * * <p>NOTE: There is seldom any need to set HOSTNAME, as it is always set implicitly (in {@link Flags}) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 1c043ad0aa6..400934fce97 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -231,6 +231,13 @@ public class Flags { "Takes effect on next tick.", NODE_TYPE); + public static final UnboundStringFlag DIST_HOST = defineStringFlag( + "dist-host", "", + List.of("freva"), "2024-04-15", "2024-05-31", + "Sets dist_host YUM variable, empty means old behavior. Only effective in Public.", + "Provisioning of instance or next host-admin tick", + HOSTNAME, NODE_TYPE, CLOUD_ACCOUNT); + public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, List.of("olaa"), "2021-09-13", "2024-09-01", @@ -343,21 +350,21 @@ public class Flags { public static final UnboundBooleanFlag MORE_WIREGUARD = defineFeatureFlag( "more-wireguard", false, - List.of("andreer"), "2023-08-21", "2024-04-14", + List.of("andreer"), "2023-08-21", "2025-01-01", "Use wireguard in INternal enCLAVES", "Takes effect on next host-admin run", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag IPV6_AWS_TARGET_GROUPS = defineFeatureFlag( "ipv6-aws-target-groups", false, - List.of("andreer"), "2023-08-28", "2024-04-14", + List.of("andreer"), "2023-08-28", "2025-01-01", "Always use IPv6 target groups for load balancers in aws", "Takes effect on next load-balancer provisioning", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag PROVISION_IPV6_ONLY_AWS = defineFeatureFlag( "provision-ipv6-only", false, - List.of("andreer"), "2023-08-28", "2024-04-14", + List.of("andreer"), "2023-08-28", "2025-01-01", "Provision without private IPv4 addresses in INternal enCLAVES in AWS", "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); @@ -370,13 +377,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag DYNAMIC_HEAP_SIZE = defineFeatureFlag( - "dynamic-heap-size", true, - List.of("bjorncs"), "2023-09-21", "2024-04-15", - "Whether to calculate JVM heap size based on predicted Onnx model memory requirements", - "Takes effect at redeployment", - INSTANCE_ID); - public static final UnboundStringFlag UNKNOWN_CONFIG_DEFINITION = defineStringFlag( "unknown-config-definition", "warn", List.of("hmusum"), "2023-09-25", "2024-09-01", @@ -423,11 +423,26 @@ public class Flags { "Takes effect immediately"); public static UnboundBooleanFlag NEW_PATH_FOR_DISK_ENCRYPTION_KEY_METADATA = defineFeatureFlag( - "new-path-for-disk-encryption-key-metadata", false, + "new-path-for-disk-encryption-key-metadata", true, List.of("hmusum"), "2024-04-08", "2024-06-01", "Whether to read and write disk encryption key to new path", "Will be read only on boot."); + public static final UnboundIntFlag PERSISTENCE_THREAD_MAX_FEED_OP_BATCH_SIZE = defineIntFlag( + "persistence-thread-max-feed-op-batch-size", 1, + List.of("vekterli"), "2024-04-12", "2025-01-01", + "Maximum number of enqueued feed operations (put/update/remove) bound "+ + "towards the same bucket that can be async dispatched as part of the " + + "same write-locked batch by a persistence thread.", + "Takes effect at redeployment", + INSTANCE_ID); + + public static UnboundBooleanFlag LOGSERVER_OTELCOL_AGENT = defineFeatureFlag( + "logserver-otelcol-agent", false, + List.of("olaa"), "2024-04-03", "2024-12-31", + "Whether logserver container should run otel agent", + "Takes effect at redeployment", INSTANCE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java index 4f191c06022..a5d88010a24 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/OrderedFlagSource.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -18,7 +17,7 @@ public class OrderedFlagSource implements FlagSource { * @param sources Flag sources in descending priority order. */ public OrderedFlagSource(FlagSource... sources) { - this.sources = Arrays.asList(sources); + this.sources = List.of(sources); } @Override diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 43bf3ec02c5..ad9f4c6a7ab 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -14,8 +14,11 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import static com.yahoo.vespa.flags.Dimension.APPLICATION; +import static com.yahoo.vespa.flags.Dimension.ARCHITECTURE; import static com.yahoo.vespa.flags.Dimension.CERTIFICATE_PROVIDER; +import static com.yahoo.vespa.flags.Dimension.CLAVE; import static com.yahoo.vespa.flags.Dimension.CLOUD_ACCOUNT; +import static com.yahoo.vespa.flags.Dimension.FLAVOR; import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; import static com.yahoo.vespa.flags.Dimension.CLUSTER_ID; import static com.yahoo.vespa.flags.Dimension.CLUSTER_TYPE; @@ -276,10 +279,19 @@ public class PermanentFlags { // This must be set in a feature flag to avoid flickering between the new and old value during config server upgrade public static final UnboundDoubleFlag HOST_MEMORY = defineDoubleFlag( - "host-memory", 0.6, - "The memory in GB required by a host's management processes.", - "Takes effect immediately" - ); + "host-memory", -1.0, + "The memory in GB required by a host's management processes. " + + "A negative value falls back to hard-coded defaults.", + "Affects future deployments, JVM settings for new config server Podman containers, auto scaling modelling.", + ARCHITECTURE, CLAVE, CLOUD_ACCOUNT, FLAVOR); + + // This must be set in a feature flag to avoid flickering between the new and old value during config server upgrade + public static final UnboundDoubleFlag HOST_MEMORY_RATIO = defineDoubleFlag( + "host-memory-ratio", -1.0, + "The ratio of MemTotal reserved for Linux or host processes, and not available to the Podman containers. " + + "A value outside the range [0.0, 1.0] will use a hard-coded ratio.", + "Affects future deployments, JVM settings for new config server Podman containers, auto scaling modelling.", + ARCHITECTURE, CLAVE, CLOUD_ACCOUNT, FLAVOR); public static final UnboundBooleanFlag FORWARD_ISSUES_AS_ERRORS = defineFeatureFlag( "forward-issues-as-errors", true, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java index c4597d1d3b4..73803e5bc5f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -14,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ public class FlagDbFile { public Map<FlagId, FlagData> read() { Optional<byte[]> bytes = readFile(); - if (!bytes.isPresent()) return Collections.emptyMap(); + if (!bytes.isPresent()) return Map.of(); return FlagData.deserializeList(bytes.get()).stream().collect(Collectors.toMap(FlagData::id, Function.identity())); } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java b/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java index 032874dffac..dcf7d758e48 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/DimensionTest.java @@ -13,7 +13,7 @@ class DimensionTest { public String remember_to_update_SystemFlagsDataArchive(Dimension dimension) { return switch (dimension) { case APPLICATION, ARCHITECTURE, CERTIFICATE_PROVIDER, CLAVE, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, - CONSOLE_USER_EMAIL, ENVIRONMENT, HOSTNAME, INSTANCE_ID, NODE_TYPE, SYSTEM, TENANT_ID, + CONSOLE_USER_EMAIL, ENVIRONMENT, FLAVOR, HOSTNAME, INSTANCE_ID, NODE_TYPE, SYSTEM, TENANT_ID, VESPA_VERSION, ZONE_ID -> dimension.toWire(); }; } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java index 4ea4f8ab638..d8530bf0a9d 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/file/FlagDbFileTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -68,7 +67,7 @@ public class FlagDbFileTest { assertThat(anotherReadDataMap, IsMapContaining.hasKey(id3)); assertThat(anotherReadDataMap.get(id1).serializeToJson(), equalTo("{\"id\":\"id1\",\"attributes\":{\"hostname\":\"h1\"}}")); - assertThat(flagDb.sync(Collections.emptyMap()), equalTo(true)); + assertThat(flagDb.sync(Map.of()), equalTo(true)); assertThat(getDbContent(), equalTo("{\"flags\":[]}")); } diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java index ca4e6f6963c..f5674b15a79 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/TestDescriptorTest.java @@ -5,7 +5,6 @@ import com.yahoo.test.json.JsonTestHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; /** @@ -31,13 +30,13 @@ public class TestDescriptorTest { Assertions.assertIterableEquals(List.of("ai.vespa.test.SystemTest1", "ai.vespa.test.SystemTest2"), systemTests); var stagingTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest); - Assertions.assertIterableEquals(Collections.emptyList(), stagingTests); + Assertions.assertIterableEquals(List.of(), stagingTests); var stagingSetupTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.stagingtest); - Assertions.assertIterableEquals(Collections.emptyList(), stagingSetupTests); + Assertions.assertIterableEquals(List.of(), stagingSetupTests); var productionTests = testClassDescriptor.getConfiguredTests(TestDescriptor.TestCategory.productiontest); - Assertions.assertIterableEquals(Collections.emptyList(), productionTests); + Assertions.assertIterableEquals(List.of(), productionTests); } @Test diff --git a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java index 57a7615421d..c17ae63fda2 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java @@ -83,7 +83,7 @@ public interface HttpClient extends Closeable { /** Sets query parameters without a value, like {@code ?debug&recursive}. */ default RequestBuilder emptyParameters(String... keys) { - return emptyParameters(Arrays.asList(keys)); + return emptyParameters(List.of(keys)); } /** Sets query parameters without a value, like {@code ?debug&recursive}. */ diff --git a/http-client/src/test/java/ai/vespa/hosted/client/ApacheHttpClientTest.java b/http-client/src/test/java/ai/vespa/hosted/client/ApacheHttpClientTest.java index a80d80bb248..69e7ab6f744 100644 --- a/http-client/src/test/java/ai/vespa/hosted/client/ApacheHttpClientTest.java +++ b/http-client/src/test/java/ai/vespa/hosted/client/ApacheHttpClientTest.java @@ -60,7 +60,7 @@ class ApacheHttpClientTest { URI.create("http://localhost:" + server.port() + "/"))), Method.GET) .at("root") - .parameters("query", "foo") + .parameters("query", "foo", null, null) .discard()); server.verify(2, getRequestedFor(urlEqualTo("/root?query=foo"))); server.verify(2, anyRequestedFor(anyUrl())); diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java index c843b77e359..0f4477b49cd 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedConnectionLevelRetryHandlerTest.java @@ -8,7 +8,7 @@ import javax.net.ssl.SSLException; import java.io.IOException; import java.net.ConnectException; import java.time.Duration; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -105,7 +105,7 @@ public class DelayedConnectionLevelRetryHandlerTest { DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder .withFixedDelay(Duration.ofSeconds(2), maxRetries) - .retryForExceptions(Arrays.asList(SSLException.class, ConnectException.class)) + .retryForExceptions(List.of(SSLException.class, ConnectException.class)) .withSleeper(mock(Sleeper.class)) .build(); @@ -122,7 +122,7 @@ public class DelayedConnectionLevelRetryHandlerTest { void does_not_retry_for_non_listed_exception() { DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder .withFixedDelay(Duration.ofSeconds(2), 2) - .retryForExceptions(Arrays.asList(SSLException.class, ConnectException.class)) + .retryForExceptions(List.of(SSLException.class, ConnectException.class)) .withSleeper(mock(Sleeper.class)) .build(); diff --git a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java index b7d15f3cd12..83c87ab3a23 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/hc4/retry/DelayedResponseLevelRetryHandlerTest.java @@ -10,7 +10,6 @@ import org.apache.http.message.BasicStatusLine; import org.junit.jupiter.api.Test; import java.time.Duration; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -84,7 +83,7 @@ public class DelayedResponseLevelRetryHandlerTest { HttpClientContext ctx = new HttpClientContext(); int lastExecutionCount = maxRetries + 1; List<Duration> expectedIntervals = - Arrays.asList( + List.of( startDelay, Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5)); @@ -100,7 +99,7 @@ public class DelayedResponseLevelRetryHandlerTest { DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder .withFixedDelay(Duration.ofSeconds(2), maxRetries) - .retryForStatusCodes(Arrays.asList(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) + .retryForStatusCodes(List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) .build(); HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE); @@ -116,7 +115,7 @@ public class DelayedResponseLevelRetryHandlerTest { void does_not_retry_for_non_listed_exception() { DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder .withFixedDelay(Duration.ofSeconds(2), 2) - .retryForStatusCodes(Arrays.asList(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) + .retryForStatusCodes(List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY)) .build(); HttpResponse response = createResponse(HttpStatus.SC_OK); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index 6e90142d8a1..c12239a99c3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -19,7 +19,7 @@ import java.util.*; public final class CatExpression extends ExpressionList<Expression> { public CatExpression(Expression... lst) { - this(Arrays.asList(lst)); + this(List.of(lst)); } public CatExpression(Collection<? extends Expression> lst) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 9038fbad33c..7a556bd90d0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -5,7 +5,6 @@ import com.yahoo.document.DataType; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -25,7 +24,7 @@ public class ChoiceExpression extends ExpressionList<Expression> { } public ChoiceExpression(Expression... choices) { - this(Arrays.asList(choices)); + this(List.of(choices)); } public ChoiceExpression(Collection<? extends Expression> choices) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java index ba07fc00ca8..cdd0c11baac 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java @@ -22,7 +22,7 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter { private final FieldValueAdapter adapter; private FieldValue value; private Language language; - private final Map<String, Object> cache = LazyMap.newHashMap(); + private final Map<Object, Object> cache = LazyMap.newHashMap(); public ExecutionContext() { this(null); @@ -125,12 +125,12 @@ public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter { } /** Returns a cached value, or null if not present. */ - public Object getCachedValue(String key) { + public Object getCachedValue(Object key) { return cache.get(key); } /** Returns a mutable reference to the cache of this. */ - public Map<String, Object> getCache() { + public Map<Object, Object> getCache() { return cache; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index 7d180b9fd7a..810ff261f2d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -12,9 +12,7 @@ import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; import com.yahoo.vespa.indexinglanguage.parser.ParseException; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -26,11 +24,11 @@ import java.util.Objects; public final class ScriptExpression extends ExpressionList<StatementExpression> { public ScriptExpression() { - this(Collections.emptyList()); + this(List.of()); } public ScriptExpression(StatementExpression... statements) { - this(Arrays.asList(statements)); + this(List.of(statements)); } public ScriptExpression(Collection<? extends StatementExpression> statements) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 3d28f70cbd5..9c92dd452c3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -25,7 +24,7 @@ public final class SelectInputExpression extends CompositeExpression { @SafeVarargs @SuppressWarnings("varargs") public SelectInputExpression(Pair<String, Expression>... cases) { - this(Arrays.asList(cases)); + this(List.of(cases)); } public SelectInputExpression(List<Pair<String, Expression>> cases) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 2db6c760380..f9d8002100f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -30,7 +30,7 @@ public final class StatementExpression extends ExpressionList<Expression> { private String outputField; public StatementExpression(Expression... lst) { - this(Arrays.asList(lst)); + this(Arrays.asList(lst)); //TODO Can contain null - necessary ? } public StatementExpression(Iterable<Expression> lst) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index 72a03e938fb..52a015137aa 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -55,7 +55,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import org.junit.Test; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -109,7 +108,7 @@ public class ExpressionConverterTestCase { assertConvertable(new StatementExpression(new InputExpression("foo"))); assertConvertable(new SubstringExpression(6, 9)); assertConvertable(new SummaryExpression("foo")); - assertConvertable(new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")), + assertConvertable(new SwitchExpression(Map.of("foo", (Expression)new IndexExpression("bar")), new InputExpression("baz"))); assertConvertable(new ThisExpression()); assertConvertable(new ToArrayExpression()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java index 4a8348204a6..c4f71402afa 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java @@ -17,8 +17,8 @@ import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -67,18 +67,18 @@ public class ExpressionSearcherTestCase { assertFound(exp, new ScriptExpression(new StatementExpression(new AttributeExpression("foo")), new StatementExpression(exp))); assertFound(exp, new SelectInputExpression( - Arrays.asList(new Pair<String, Expression>("foo", exp), - new Pair<String, Expression>("bar", new AttributeExpression("bar"))))); + List.of(new Pair<>("foo", exp), + new Pair<>("bar", new AttributeExpression("bar"))))); assertFound(exp, new SelectInputExpression( - Arrays.asList(new Pair<String, Expression>("foo", new AttributeExpression("bar")), - new Pair<String, Expression>("bar", exp)))); + List.of(new Pair<>("foo", new AttributeExpression("bar")), + new Pair<>("bar", exp)))); assertFound(exp, new StatementExpression(exp)); assertFound(exp, new StatementExpression(new AttributeExpression("foo"), exp)); assertFound(exp, new SwitchExpression( - Collections.singletonMap("foo", exp), + Map.of("foo", exp), new AttributeExpression("bar"))); assertFound(exp, new SwitchExpression( - Collections.singletonMap("foo", new AttributeExpression("bar")), + Map.of("foo", new AttributeExpression("bar")), exp)); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java index 5bbf62bfff7..de512997b64 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java @@ -55,7 +55,8 @@ import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import org.junit.Test; -import java.util.Collections; + +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -64,7 +65,6 @@ import static org.junit.Assert.assertEquals; */ public class ExpressionVisitorTestCase { - @SuppressWarnings("unchecked") @Test public void requireThatAllExpressionsAreVisited() { assertCount(3, new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, @@ -96,8 +96,8 @@ public class ExpressionVisitorTestCase { assertCount(2, new ParenthesisExpression(new InputExpression("foo"))); assertCount(1, new RandomExpression(69)); assertCount(3, new ScriptExpression(new StatementExpression(new InputExpression("foo")))); - assertCount(3, new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), - new Pair<String, Expression>("bar", new IndexExpression("foo")))); + assertCount(3, new SelectInputExpression(new Pair<>("foo", new IndexExpression("bar")), + new Pair<>("bar", new IndexExpression("foo")))); assertCount(1, new SetLanguageExpression()); assertCount(1, new ConstantExpression(new IntegerFieldValue(69))); assertCount(1, new SetVarExpression("foo")); @@ -105,7 +105,7 @@ public class ExpressionVisitorTestCase { assertCount(2, new StatementExpression(new InputExpression("foo"))); assertCount(1, new SummaryExpression("foo")); assertCount(1, new SubstringExpression(6, 9)); - assertCount(3, new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")), + assertCount(3, new SwitchExpression(Map.of("foo", (Expression)new IndexExpression("bar")), new InputExpression("baz"))); assertCount(1, new ThisExpression()); assertCount(1, new ToArrayExpression()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java index 6403bbb24ac..17388f65656 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java @@ -7,7 +7,8 @@ import com.yahoo.document.datatypes.*; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.util.Arrays; + +import java.util.List; import static org.junit.Assert.*; @@ -25,7 +26,7 @@ public class CatTestCase { assertEquals(2, exp.size()); assertSame(foo, exp.get(0)); assertSame(bar, exp.get(1)); - assertEquals(Arrays.asList(foo, bar), exp.asList()); + assertEquals(List.of(foo, bar), exp.asList()); } @Test @@ -33,11 +34,11 @@ public class CatTestCase { Expression foo = new AttributeExpression("foo"); Expression bar = new AttributeExpression("bar"); Expression exp = new CatExpression(foo, bar); - assertFalse(exp.equals(new Object())); - assertFalse(exp.equals(new StatementExpression(foo, bar))); - assertFalse(exp.equals(new CatExpression())); - assertFalse(exp.equals(new CatExpression(foo))); - assertFalse(exp.equals(new CatExpression(bar, foo))); + assertNotEquals(exp, new Object()); + assertNotEquals(exp, new StatementExpression(foo, bar)); + assertNotEquals(exp, new CatExpression()); + assertNotEquals(exp, new CatExpression(foo)); + assertNotEquals(exp, new CatExpression(bar, foo)); assertEquals(exp, new CatExpression(foo, bar)); assertEquals(exp.hashCode(), new CatExpression(foo, bar).hashCode()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java index 287398de030..c45b16beadb 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.document.serialization.XmlStream; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; @@ -174,12 +173,12 @@ public class IfThenTestCase { @Test public void requireThatNumericValuesSupportNumericCompareTo() { - List<NumericFieldValue> sixes = Arrays.asList(new ByteFieldValue((byte)6), + List<NumericFieldValue> sixes = List.of(new ByteFieldValue((byte)6), new DoubleFieldValue(6.0), new FloatFieldValue(6.0f), new IntegerFieldValue(6), new LongFieldValue(6L)); - List<NumericFieldValue> nines = Arrays.asList(new ByteFieldValue((byte)9), + List<NumericFieldValue> nines = List.of(new ByteFieldValue((byte)9), new DoubleFieldValue(9.0), new FloatFieldValue(9.0f), new IntegerFieldValue(9), diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java index df7f99d22d2..c6fb7c1db8c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java @@ -8,7 +8,8 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; @@ -32,7 +33,7 @@ public class ScriptTestCase { assertEquals(2, exp.size()); assertSame(foo, exp.get(0)); assertSame(bar, exp.get(1)); - assertEquals(Arrays.asList(foo, bar), exp.asList()); + assertEquals(List.of(foo, bar), exp.asList()); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java index 2c10606850c..23392c00110 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -66,11 +65,11 @@ public class SelectInputTestCase { @Test public void requireThatSelectedExpressionIsRun() { - assertSelect(Arrays.asList("foo", "bar"), List.of("foo"), "foo"); - assertSelect(Arrays.asList("foo", "bar"), List.of("bar"), "bar"); - assertSelect(Arrays.asList("foo", "bar"), List.of("foo", "bar"), "foo"); - assertSelect(Arrays.asList("foo", "bar"), List.of("bar", "baz"), "bar"); - assertSelect(Arrays.asList("foo", "bar"), List.of("baz", "cox"), null); + assertSelect(List.of("foo", "bar"), List.of("foo"), "foo"); + assertSelect(List.of("foo", "bar"), List.of("bar"), "bar"); + assertSelect(List.of("foo", "bar"), List.of("foo", "bar"), "foo"); + assertSelect(List.of("foo", "bar"), List.of("bar", "baz"), "bar"); + assertSelect(List.of("foo", "bar"), List.of("baz", "cox"), null); } private static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java index 7e5b17dbc37..3612e3bd9fc 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java @@ -7,7 +7,8 @@ import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; @@ -31,7 +32,7 @@ public class StatementTestCase { assertEquals(2, exp.size()); assertSame(foo, exp.get(0)); assertSame(bar, exp.get(1)); - assertEquals(Arrays.asList(foo, bar), exp.asList()); + assertEquals(List.of(foo, bar), exp.asList()); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java index 62d333e356f..376677b6f57 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java @@ -7,7 +7,6 @@ import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import org.junit.Test; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -45,10 +44,10 @@ public class SwitchTestCase { cases.put("bar", bar); SwitchExpression exp = new SwitchExpression(cases, baz); - assertFalse(exp.equals(new Object())); - assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo)))); - assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo), baz))); - assertFalse(exp.equals(new SwitchExpression(cases))); + assertNotEquals(exp, new Object()); + assertNotEquals(exp, new SwitchExpression(Map.of("foo", foo))); + assertNotEquals(exp, new SwitchExpression(Map.of("foo", foo), baz)); + assertNotEquals(exp, new SwitchExpression(cases)); assertEquals(exp, new SwitchExpression(cases, baz)); assertEquals(exp.hashCode(), new SwitchExpression(cases, baz).hashCode()); } @@ -56,7 +55,7 @@ public class SwitchTestCase { @Test public void requireThatExpressionCanBeVerified() { Expression foo = SimpleExpression.newConversion(DataType.STRING, DataType.INT); - Expression exp = new SwitchExpression(Collections.singletonMap("foo", foo)); + Expression exp = new SwitchExpression(Map.of("foo", foo)); assertVerify(DataType.STRING, exp, DataType.STRING); // does not touch output assertVerifyThrows(null, exp, "Expected string input, but no input is specified"); assertVerifyThrows(DataType.INT, exp, "Expected string input, got int"); @@ -68,15 +67,14 @@ public class SwitchTestCase { cases.put("foo", SimpleExpression.newRequired(DataType.INT)); assertVerifyThrows(DataType.STRING, new SwitchExpression(cases), "Expected int input, got string"); - assertVerifyThrows(DataType.STRING, new SwitchExpression(Collections.<String, Expression>emptyMap(), - SimpleExpression.newRequired(DataType.INT)), + assertVerifyThrows(DataType.STRING, new SwitchExpression(Map.of(), SimpleExpression.newRequired(DataType.INT)), "Expected int input, got string"); } @Test public void requireThatIllegalArgumentThrows() { try { - new SwitchExpression(Collections.<String, Expression>emptyMap()).execute(new IntegerFieldValue(69)); + new SwitchExpression(Map.of()).execute(new IntegerFieldValue(69)); fail(); } catch (IllegalArgumentException e) { assertEquals("Expected string input, got int", e.getMessage()); @@ -85,18 +83,18 @@ public class SwitchTestCase { @Test public void requireThatDefaultExpressionIsNullIfNotGiven() { - assertNull(new SwitchExpression(Collections.<String, Expression>emptyMap()).getDefaultExpression()); + assertNull(new SwitchExpression(Map.of()).getDefaultExpression()); } @Test public void requireThatIsEmptyReflectsOnBothCasesAndDefault() { - assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap()).isEmpty()); - assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap(), null).isEmpty()); - assertFalse(new SwitchExpression(Collections.<String, Expression>emptyMap(), + assertTrue(new SwitchExpression(Map.of()).isEmpty()); + assertTrue(new SwitchExpression(Map.of(), null).isEmpty()); + assertFalse(new SwitchExpression(Map.of(), new AttributeExpression("foo")).isEmpty()); - assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")), + assertFalse(new SwitchExpression(Map.of("foo", new AttributeExpression("foo")), null).isEmpty()); - assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")), + assertFalse(new SwitchExpression(Map.of("foo", new AttributeExpression("foo")), new AttributeExpression("foo")).isEmpty()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 0bdf98f2ae0..136e71564d8 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -18,7 +18,9 @@ import com.yahoo.language.simple.SimpleToken; import org.junit.Test; import org.mockito.Mockito; -import java.util.*; + +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -90,7 +92,7 @@ public class LinguisticsAnnotatorTestCase { public void requireThatTermAnnotationsAreEmptyIfOrigIsLowerCase() { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM)); - for (boolean specialToken : Arrays.asList(true, false)) { + for (boolean specialToken : List.of(true, false)) { for (TokenType type : TokenType.values()) { if (!specialToken && !type.isIndexable()) { continue; @@ -104,7 +106,7 @@ public class LinguisticsAnnotatorTestCase { public void requireThatTermAnnotationsPreserveCasing() { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("BaR"))); - for (boolean specialToken : Arrays.asList(true, false)) { + for (boolean specialToken : List.of(true, false)) { for (TokenType type : TokenType.values()) { if (!specialToken && !type.isIndexable()) { continue; @@ -153,7 +155,7 @@ public class LinguisticsAnnotatorTestCase { public void requireThatTermReplacementsAreApplied() { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); - for (boolean specialToken : Arrays.asList(true, false)) { + for (boolean specialToken : List.of(true, false)) { for (TokenType type : TokenType.values()) { if (!specialToken && !type.isIndexable()) { continue; @@ -161,7 +163,7 @@ public class LinguisticsAnnotatorTestCase { assertAnnotations(expected, "foo", new AnnotatorConfig(), newLinguistics(List.of(token("foo", "foo", type, specialToken)), - Collections.singletonMap("foo", "bar"))); + Map.of("foo", "bar"))); } } } @@ -175,7 +177,7 @@ public class LinguisticsAnnotatorTestCase { val.setSpanTree(spanTree); Linguistics linguistics = newLinguistics(List.of(token("foo", "bar", TokenType.ALPHABETIC, false)), - Collections.<String, String>emptyMap()); + Map.of()); assertTrue(new LinguisticsAnnotator(linguistics, new AnnotatorConfig()).annotate(val)); assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS)); } @@ -244,11 +246,11 @@ public class LinguisticsAnnotatorTestCase { } private static void assertAnnotations(SpanTree expected, String value, Token... tokens) { - assertAnnotations(expected, value, new AnnotatorConfig(), newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); + assertAnnotations(expected, value, new AnnotatorConfig(), newLinguistics(List.of(tokens), Map.of())); } private static void assertAnnotations(SpanTree expected, String value, AnnotatorConfig config, Token... tokens) { - assertAnnotations(expected, value, config, newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); + assertAnnotations(expected, value, config, newLinguistics(List.of(tokens), Map.of())); } private static void assertAnnotations(SpanTree expected, String str, AnnotatorConfig config, Linguistics linguistics) { @@ -272,9 +274,12 @@ public class LinguisticsAnnotatorTestCase { } private Token replace(Token token, Map<String, String> replacementTerms) { - var simpleToken = (SimpleToken)token; - simpleToken.setTokenString(replacementTerms.getOrDefault(token.getTokenString(), token.getTokenString())); - return simpleToken; + String tokenString = token.getTokenString(); + if (tokenString != null && !replacementTerms.isEmpty()) { + var simpleToken = (SimpleToken)token; + simpleToken.setTokenString(replacementTerms.getOrDefault(token.getTokenString(), token.getTokenString())); + } + return token; } @Override diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java index 300ae330e11..4ac5c840536 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.indexinglanguage.parser; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -15,7 +14,7 @@ public class IdentifierTestCase { @Test public void requireThatThereAreNoReservedWords() throws ParseException { - List<String> tokens = Arrays.asList("attribute", + List<String> tokens = List.of("attribute", "base64decode", "base64encode", "clear_state", diff --git a/integration/intellij/build.gradle.kts b/integration/intellij/build.gradle.kts index 89101020920..6fff4e8f519 100644 --- a/integration/intellij/build.gradle.kts +++ b/integration/intellij/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } group="ai.vespa" -version="1.6.0" // Also update pom.xml version AND the version below if this is changed +version="1.6.1" // Also update pom.xml version AND the version below if this is changed defaultTasks("buildPlugin") @@ -18,14 +18,13 @@ apply(plugin="org.jetbrains.grammarkit") task<GenerateLexerTask>("generateSdLexer") { sourceFile.set(file("src/main/jflex/ai/vespa/intellij/schema/lexer/sd.flex")) - targetDir.set("target/generated-sources/jflex/ai/vespa/intellij/schema/lexer/") - targetClass.set("SdLexer") + targetOutputDir.set(file("target/generated-sources/jflex/ai/vespa/intellij/schema/lexer/")) purgeOldFiles.set(true) } task<GenerateParserTask>("generateSdParser") { sourceFile.set(file("src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf")) - targetRoot.set("target/generated-sources/bnf/") + targetRootOutputDir.set(file("target/generated-sources/bnf/")) pathToParser.set("ai/vespa/intellij/schema/parser/SdParser.java") pathToPsiRoot.set("ai/vespa/intellij/schema/parser/psi/") purgeOldFiles.set(true) @@ -43,7 +42,7 @@ sourceSets { // See https://github.com/JetBrains/gradle-intellij-plugin/ intellij { - version.set("2023.3") + version.set("2024.1") } tasks { @@ -55,12 +54,10 @@ tasks { } patchPluginXml { - version.set("1.6.0") // Keep in sync with pom.xml TODO: Use one version property + version.set("1.6.1") // Keep in sync with pom.xml TODO: Use one version property // Appears on the plugin page in preferences/plugins changeNotes.set(""" - Updated Vespa icon - Support for IntelliJ 2023.3 - Compatibility with all JetBrains IDEs + Support for IntelliJ 2024.1 """) } diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml index 8156ac8a142..4a82c73f98e 100644 --- a/integration/intellij/pom.xml +++ b/integration/intellij/pom.xml @@ -9,7 +9,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> - <version>1.6.0</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle.kts --> + <version>1.6.1</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle.kts --> <description> Maven wrapper for the gradle build of this IntelliJ plugin. </description> diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java index 96306f60d80..83e2a8ea4f0 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java @@ -11,7 +11,8 @@ import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig.Builder; import com.yahoo.jdisc.http.filter.util.FilterTestUtils; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; import static com.yahoo.jdisc.http.filter.security.cors.CorsLogic.ACCESS_CONTROL_HEADERS; @@ -75,7 +76,7 @@ public class CorsPreflightRequestFilterTest { private static CorsPreflightRequestFilter newRequestFilter(String... allowedOriginUrls) { Builder builder = new Builder(); - Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls); + List.of(allowedOriginUrls).forEach(builder::allowedUrls); return new CorsPreflightRequestFilter(new CorsFilterConfig(builder)); } diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilterTest.java index a8fb8c8d817..8c28e0c2a98 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsResponseFilterTest.java @@ -8,9 +8,9 @@ import com.yahoo.jdisc.http.filter.SecurityResponseFilter; import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig.Builder; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,7 +68,7 @@ public class CorsResponseFilterTest { private static CorsResponseFilter newResponseFilter(String... allowedOriginUrls) { Builder builder = new Builder(); - Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls); + List.of(allowedOriginUrls).forEach(builder::allowedUrls); return new CorsResponseFilter(new CorsFilterConfig(builder)); } diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java index 30af4a9ac23..1da5b1b39a3 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java @@ -17,7 +17,6 @@ import java.math.BigInteger; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -28,7 +27,7 @@ public class VespaTlsFilterTest { @Test void testFilter() { assertSuccess(createRequest(List.of(createCertificate()))); - assertForbidden(createRequest(Collections.emptyList())); + assertForbidden(createRequest(List.of())); } private static X509Certificate createCertificate() { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java index 71956691623..3485c07c74e 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/AbstractApplication.java @@ -6,7 +6,6 @@ import com.yahoo.jdisc.service.CurrentContainer; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; -import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -67,7 +66,7 @@ public abstract class AbstractApplication implements Application { } public final List<Bundle> installAndStartBundle(String... locations) throws BundleException { - return installAndStartBundle(Arrays.asList(locations)); + return installAndStartBundle(List.of(locations)); } public final List<Bundle> installAndStartBundle(Iterable<String> locations) throws BundleException { @@ -75,7 +74,7 @@ public abstract class AbstractApplication implements Application { } public final void stopAndUninstallBundle(Bundle... bundles) throws BundleException { - stopAndUninstallBundle(Arrays.asList(bundles)); + stopAndUninstallBundle(List.of(bundles)); } public final void stopAndUninstallBundle(Iterable<Bundle> bundles) throws BundleException { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BundleInstaller.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BundleInstaller.java index 60ca0851fd4..1e0050335dc 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BundleInstaller.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BundleInstaller.java @@ -5,11 +5,9 @@ import com.google.inject.Inject; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; -import static java.util.Collections.singletonList; /** * <p>This is a utility class to help with installing, starting, stopping and uninstalling OSGi Bundles. You can choose @@ -30,7 +28,7 @@ public final class BundleInstaller { } public List<Bundle> installAndStart(String... locations) throws BundleException { - return installAndStart(Arrays.asList(locations)); + return installAndStart(List.of(locations)); } public List<Bundle> installAndStart(Iterable<String> locations) throws BundleException { @@ -56,7 +54,7 @@ public final class BundleInstaller { } public void stopAndUninstall(Bundle... bundles) throws BundleException { - stopAndUninstall(Arrays.asList(bundles)); + stopAndUninstall(List.of(bundles)); } public void stopAndUninstall(Iterable<Bundle> bundles) throws BundleException { @@ -76,7 +74,7 @@ public final class BundleInstaller { throw new BundleException("OSGi header '" + OsgiHeader.APPLICATION + "' not allowed for " + "non-application bundle " + bundle.getSymbolicName() + "."); } - osgiFramework.startBundles(singletonList(bundle), false); + osgiFramework.startBundles(List.of(bundle), false); } private void stop(Bundle bundle) throws BundleException { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java index 6550d9b5386..8b3fe4c13aa 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ContainerBuilder.java @@ -8,7 +8,6 @@ import com.google.inject.Module; import com.yahoo.jdisc.Container; import com.yahoo.jdisc.handler.RequestHandler; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -107,7 +106,7 @@ public class ContainerBuilder { public static List<String> safeStringSplit(Object obj, String delim) { if (!(obj instanceof String)) { - return Collections.emptyList(); + return List.of(); } List<String> lst = new LinkedList<>(); for (String str : ((String)obj).split(delim)) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/GuiceRepository.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/GuiceRepository.java index fb3965108d8..5fc3b42055f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/GuiceRepository.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/GuiceRepository.java @@ -13,7 +13,6 @@ import com.google.inject.spi.Elements; import com.yahoo.jdisc.Container; import org.osgi.framework.Bundle; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -38,7 +37,7 @@ public class GuiceRepository implements Iterable<Module> { private Injector injector; public GuiceRepository(Module... modules) { - installAll(Arrays.asList(modules)); + installAll(List.of(modules)); } public Injector activate() { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/client/ClientDriver.java b/jdisc_core/src/main/java/com/yahoo/jdisc/client/ClientDriver.java index 8a5c8de92b9..beba03d7d84 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/client/ClientDriver.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/client/ClientDriver.java @@ -10,7 +10,6 @@ import com.yahoo.jdisc.core.FelixFramework; import com.yahoo.jdisc.core.FelixParams; import com.yahoo.jdisc.test.NonWorkingOsgiFramework; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -94,7 +93,7 @@ public abstract class ClientDriver { } private static List<Module> newModuleList(final ClientApplication appInstance, Module... guiceModules) { - List<Module> lst = new LinkedList<>(Arrays.asList(guiceModules)); + List<Module> lst = new LinkedList<>(List.of(guiceModules)); lst.add(new AbstractModule() { @Override @@ -108,7 +107,7 @@ public abstract class ClientDriver { private static List<Module> newModuleList(final Class<? extends ClientApplication> appClass, Module... guiceModules) { - List<Module> lst = new LinkedList<>(Arrays.asList(guiceModules)); + List<Module> lst = new LinkedList<>(List.of(guiceModules)); lst.add(new AbstractModule() { @Override diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java index 80e531c90af..2bf3e05baed 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java @@ -14,7 +14,6 @@ import org.osgi.framework.FrameworkEvent; import org.osgi.framework.wiring.FrameworkWiring; import java.io.File; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; @@ -150,14 +149,14 @@ public class FelixFramework implements OsgiFramework { @Override public List<Bundle> bundles() { - return Arrays.asList(felix.getBundleContext().getBundles()); + return List.of(felix.getBundleContext().getBundles()); } @Override public List<Bundle> getBundles(Bundle requestingBundle) { log.fine(() -> "All bundles: " + bundles()); log.fine(() -> "Getting visible bundles for bundle " + requestingBundle); - List<Bundle> visibleBundles = Arrays.asList(requestingBundle.getBundleContext().getBundles()); + List<Bundle> visibleBundles = List.of(requestingBundle.getBundleContext().getBundles()); log.fine(() -> "Visible bundles: " + visibleBundles); return visibleBundles; } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java index edd4dd40496..b2a8660e2a8 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/Main.java @@ -6,8 +6,7 @@ import com.yahoo.jdisc.application.ContainerBuilder; import com.yahoo.jdisc.application.OsgiFramework; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; /** * @author Simon Thoresen Hult @@ -31,7 +30,7 @@ public class Main { static Iterable<Module> newConfigModule() { String configFile = System.getProperty("jdisc.config.file"); if (configFile == null) { - return Collections.emptyList(); + return List.of(); } Module configModule; try { @@ -39,7 +38,7 @@ public class Main { } catch (IOException e) { throw new IllegalStateException("Exception thrown while reading config file '" + configFile + "'.", e); } - return Arrays.asList(configModule); + return List.of(configModule); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java index ef420d69f16..c39a70c7093 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java @@ -8,7 +8,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.SharedResource; import java.nio.ByteBuffer; -import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; @@ -35,7 +35,7 @@ import java.util.concurrent.TimeoutException; * } * @Override * protected Iterable<ByteBuffer> requestContent() { - * return Collections.singleton(ByteBuffer.wrap(new byte[] { 6, 9 })); + * return Set.of(ByteBuffer.wrap(new byte[] { 6, 9 })); * } * @Override * public ContentChannel handleResponse(Response response) { @@ -69,7 +69,7 @@ public abstract class RequestDispatch implements Future<Response>, ResponseHandl * @return The ByteBuffers to write to the Request's ContentChannel. */ protected Iterable<ByteBuffer> requestContent() { - return Collections.emptyList(); + return List.of(); } /** diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java index ff1f23917d7..0670aa5ea67 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java @@ -5,8 +5,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.SharedResource; import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -28,7 +27,7 @@ import java.util.concurrent.TimeoutException; * } * @Override * protected Iterable<ByteBuffer> responseContent() { - * return Collections.singleton(ByteBuffer.wrap(new byte[] { 6, 9 })); + * return Set.of(ByteBuffer.wrap(new byte[] { 6, 9 })); * } * }.dispatch(handler); * } @@ -56,7 +55,7 @@ public abstract class ResponseDispatch implements Future<Boolean> { * @return The ByteBuffers to write to the Response's ContentChannel. */ protected Iterable<ByteBuffer> responseContent() { - return Collections.emptyList(); + return List.of(); } /** @@ -130,7 +129,7 @@ public abstract class ResponseDispatch implements Future<Boolean> { * @return The created ResponseDispatch. */ public static ResponseDispatch newInstance(int responseStatus, ByteBuffer... content) { - return newInstance(new Response(responseStatus), Arrays.asList(content)); + return newInstance(new Response(responseStatus), List.of(content)); } /** @@ -155,7 +154,7 @@ public abstract class ResponseDispatch implements Future<Boolean> { * @return The created ResponseDispatch. */ public static ResponseDispatch newInstance(Response response, ByteBuffer... content) { - return newInstance(response, Arrays.asList(content)); + return newInstance(response, List.of(content)); } /** diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/NonWorkingOsgiFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/NonWorkingOsgiFramework.java index 1b90e8adcfe..3565a1ec4c5 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/NonWorkingOsgiFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/NonWorkingOsgiFramework.java @@ -6,7 +6,6 @@ import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import java.util.Collection; -import java.util.Collections; import java.util.List; /** @@ -36,12 +35,12 @@ public class NonWorkingOsgiFramework implements OsgiFramework { @Override public List<Bundle> bundles() { - return Collections.emptyList(); + return List.of(); } @Override public List<Bundle> getBundles(Bundle requestingBundle) { - return Collections.emptyList(); + return List.of(); } @Override diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestDriver.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestDriver.java index 18449f207ee..64c690e89d4 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestDriver.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestDriver.java @@ -26,7 +26,6 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import java.net.URI; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.Callable; @@ -311,7 +310,7 @@ public class TestDriver implements ContainerActivator, CurrentContainer { */ public static TestDriver newApplicationBundleInstance(String bundleLocation, boolean privileged, Module... guiceModules) { - return newInstance(newOsgiFramework(), bundleLocation, privileged, Arrays.asList(guiceModules)); + return newInstance(newOsgiFramework(), bundleLocation, privileged, List.of(guiceModules)); } /** @@ -326,7 +325,7 @@ public class TestDriver implements ContainerActivator, CurrentContainer { */ public static TestDriver newInstance(OsgiFramework osgiFramework, String bundleLocation, boolean privileged, Module... guiceModules) { - return newInstance(osgiFramework, bundleLocation, privileged, Arrays.asList(guiceModules)); + return newInstance(osgiFramework, bundleLocation, privileged, List.of(guiceModules)); } /** @@ -381,7 +380,7 @@ public class TestDriver implements ContainerActivator, CurrentContainer { private static List<Module> newModuleList(final Application app, final Class<? extends Application> appClass, Module... guiceModules) { List<Module> lst = new LinkedList<>(); - lst.addAll(Arrays.asList(guiceModules)); + lst.addAll(List.of(guiceModules)); lst.add(new AbstractModule() { @Override diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/HeaderFieldsTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/HeaderFieldsTestCase.java index 7fe3c052566..acaed01a946 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/HeaderFieldsTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/HeaderFieldsTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.jdisc; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -64,9 +63,9 @@ public class HeaderFieldsTestCase { @Test void requireThatContainsValueWorksAsExpected() { HeaderFields headers = new HeaderFields(); - assertFalse(headers.containsValue(Arrays.asList("bar"))); + assertFalse(headers.containsValue(List.of("bar"))); headers.add("foo", "bar"); - assertTrue(headers.containsValue(Arrays.asList("bar"))); + assertTrue(headers.containsValue(List.of("bar"))); } @Test @@ -102,19 +101,19 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); assertNull(headers.get("foo")); headers.add("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); headers.add("foo", "baz"); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); } @Test void requireThatAddListWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertNull(headers.get("foo")); - headers.add("foo", Arrays.asList("bar")); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - headers.add("foo", Arrays.asList("baz", "cox")); - assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); + headers.add("foo", List.of("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + headers.add("foo", List.of("baz", "cox")); + assertEquals(List.of("bar", "baz", "cox"), headers.get("foo")); } @Test @@ -122,16 +121,16 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); Map<String, List<String>> map = new HashMap<>(); - map.put("foo", Arrays.asList("baz", "cox")); - map.put("bar", Arrays.asList("cox")); + map.put("foo", List.of("baz", "cox")); + map.put("bar", List.of("cox")); headers.addAll(map); - assertEquals(Arrays.asList("bar", "baz", "cox"), headers.get("foo")); - assertEquals(Arrays.asList("baz", "cox"), headers.get("bar")); + assertEquals(List.of("bar", "baz", "cox"), headers.get("foo")); + assertEquals(List.of("baz", "cox"), headers.get("bar")); } @Test @@ -139,19 +138,19 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); assertNull(headers.get("foo")); headers.put("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); headers.put("foo", "baz"); - assertEquals(Arrays.asList("baz"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("foo")); } @Test void requireThatPutListWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertNull(headers.get("foo")); - headers.put("foo", Arrays.asList("bar")); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - headers.put("foo", Arrays.asList("baz", "cox")); - assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); + headers.put("foo", List.of("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + headers.put("foo", List.of("baz", "cox")); + assertEquals(List.of("baz", "cox"), headers.get("foo")); } @Test @@ -159,24 +158,24 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); Map<String, List<String>> map = new HashMap<>(); - map.put("foo", Arrays.asList("baz", "cox")); - map.put("bar", Arrays.asList("cox")); + map.put("foo", List.of("baz", "cox")); + map.put("bar", List.of("cox")); headers.putAll(map); - assertEquals(Arrays.asList("baz", "cox"), headers.get("foo")); - assertEquals(Arrays.asList("cox"), headers.get("bar")); + assertEquals(List.of("baz", "cox"), headers.get("foo")); + assertEquals(List.of("cox"), headers.get("bar")); } @Test void requireThatRemoveWorksAsExpected() { HeaderFields headers = new HeaderFields(); - headers.put("foo", Arrays.asList("bar", "baz")); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); - assertEquals(Arrays.asList("bar", "baz"), headers.remove("foo")); + headers.put("foo", List.of("bar", "baz")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); + assertEquals(List.of("bar", "baz"), headers.remove("foo")); assertNull(headers.get("foo")); assertNull(headers.remove("foo")); } @@ -184,11 +183,11 @@ public class HeaderFieldsTestCase { @Test void requireThatRemoveStringWorksAsExpected() { HeaderFields headers = new HeaderFields(); - headers.put("foo", Arrays.asList("bar", "baz")); - assertEquals(Arrays.asList("bar", "baz"), headers.get("foo")); + headers.put("foo", List.of("bar", "baz")); + assertEquals(List.of("bar", "baz"), headers.get("foo")); assertTrue(headers.remove("foo", "bar")); assertFalse(headers.remove("foo", "cox")); - assertEquals(Arrays.asList("baz"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("foo")); assertTrue(headers.remove("foo", "baz")); assertFalse(headers.remove("foo", "cox")); assertNull(headers.get("foo")); @@ -199,8 +198,8 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); headers.add("foo", "bar"); headers.add("bar", "baz"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); - assertEquals(Arrays.asList("baz"), headers.get("bar")); + assertEquals(List.of("bar"), headers.get("foo")); + assertEquals(List.of("baz"), headers.get("bar")); headers.clear(); assertNull(headers.get("foo")); assertNull(headers.get("bar")); @@ -211,14 +210,14 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); assertNull(headers.get("foo")); headers.add("foo", "bar"); - assertEquals(Arrays.asList("bar"), headers.get("foo")); + assertEquals(List.of("bar"), headers.get("foo")); } @Test void requireThatGetFirstWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertNull(headers.getFirst("foo")); - headers.add("foo", Arrays.asList("bar", "baz")); + headers.add("foo", List.of("bar", "baz")); assertEquals("bar", headers.getFirst("foo")); } @@ -226,17 +225,17 @@ public class HeaderFieldsTestCase { void requireThatIsTrueWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true")); + headers.put("foo", List.of("true")); assertTrue(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true", "true")); + headers.put("foo", List.of("true", "true")); assertTrue(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("true", "false")); + headers.put("foo", List.of("true", "false")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false", "true")); + headers.put("foo", List.of("false", "true")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false", "false")); + headers.put("foo", List.of("false", "false")); assertFalse(headers.isTrue("foo")); - headers.put("foo", Arrays.asList("false")); + headers.put("foo", List.of("false")); assertFalse(headers.isTrue("foo")); } @@ -245,9 +244,9 @@ public class HeaderFieldsTestCase { HeaderFields headers = new HeaderFields(); assertTrue(headers.keySet().isEmpty()); headers.add("foo", "bar"); - assertEquals(new HashSet<>(Arrays.asList("foo")), headers.keySet()); + assertEquals(new HashSet<>(List.of("foo")), headers.keySet()); headers.add("bar", "baz"); - assertEquals(new HashSet<>(Arrays.asList("foo", "bar")), headers.keySet()); + assertEquals(new HashSet<>(List.of("foo", "bar")), headers.keySet()); } @Test @@ -257,34 +256,34 @@ public class HeaderFieldsTestCase { headers.add("foo", "bar"); Collection<List<String>> values = headers.values(); assertEquals(1, values.size()); - assertTrue(values.contains(Arrays.asList("bar"))); + assertTrue(values.contains(List.of("bar"))); headers.add("bar", "baz"); values = headers.values(); assertEquals(2, values.size()); - assertTrue(values.contains(Arrays.asList("bar"))); - assertTrue(values.contains(Arrays.asList("baz"))); + assertTrue(values.contains(List.of("bar"))); + assertTrue(values.contains(List.of("baz"))); } @Test void requireThatEntrySetWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertTrue(headers.entrySet().isEmpty()); - headers.put("foo", Arrays.asList("bar", "baz")); + headers.put("foo", List.of("bar", "baz")); Set<Map.Entry<String, List<String>>> entries = headers.entrySet(); assertEquals(1, entries.size()); Map.Entry<String, List<String>> entry = entries.iterator().next(); assertNotNull(entry); assertEquals("foo", entry.getKey()); - assertEquals(Arrays.asList("bar", "baz"), entry.getValue()); + assertEquals(List.of("bar", "baz"), entry.getValue()); } @Test void requireThatEntriesWorksAsExpected() { HeaderFields headers = new HeaderFields(); assertTrue(headers.entries().isEmpty()); - headers.put("foo", Arrays.asList("bar", "baz")); + headers.put("foo", List.of("bar", "baz")); List<Map.Entry<String, String>> entries = headers.entries(); assertEquals(2, entries.size()); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingSetTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingSetTestCase.java index e35f1c69073..133472f5678 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingSetTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BindingSetTestCase.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -480,13 +479,13 @@ public class BindingSetTestCase { for (int i = 0; i < expected.length; ++i) { actual.add(expected[(off + i) % expected.length]); } - assertOrder(Arrays.asList(expected), actual); + assertOrder(List.of(expected), actual); actual = new ArrayList<>(); for (int i = expected.length; --i >= 0; ) { actual.add(expected[(off + i) % expected.length]); } - assertOrder(Arrays.asList(expected), actual); + assertOrder(List.of(expected), actual); } } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BundleInstallationExceptionTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BundleInstallationExceptionTestCase.java index 6a704b1aacc..34ac6603e62 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/BundleInstallationExceptionTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/BundleInstallationExceptionTestCase.java @@ -5,9 +5,9 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.osgi.framework.Bundle; -import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; @@ -41,7 +41,7 @@ public class BundleInstallationExceptionTestCase { @Test void requireThatBundlesCollectionIsUnmodifiable() { - BundleInstallationException e = new BundleInstallationException(Arrays.asList(Mockito.mock(Bundle.class)), + BundleInstallationException e = new BundleInstallationException(List.of(Mockito.mock(Bundle.class)), new Throwable()); try { e.installedBundles().add(Mockito.mock(Bundle.class)); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/ContainerBuilderTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/ContainerBuilderTestCase.java index 9b3d61d7c89..3737647679d 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/ContainerBuilderTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/ContainerBuilderTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.jdisc.test.TestDriver; import org.junit.jupiter.api.Test; import java.net.URISyntaxException; -import java.util.Arrays; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -106,11 +106,11 @@ public class ContainerBuilderTestCase { assertTrue(ContainerBuilder.safeStringSplit(new Object(), ",").isEmpty()); assertTrue(ContainerBuilder.safeStringSplit("", ",").isEmpty()); assertTrue(ContainerBuilder.safeStringSplit(" \f\n\r\t", ",").isEmpty()); - assertEquals(Arrays.asList("foo"), ContainerBuilder.safeStringSplit("foo", ",")); - assertEquals(Arrays.asList("foo"), ContainerBuilder.safeStringSplit(" foo", ",")); - assertEquals(Arrays.asList("foo"), ContainerBuilder.safeStringSplit("foo ", ",")); - assertEquals(Arrays.asList("foo"), ContainerBuilder.safeStringSplit("foo, ", ",")); - assertEquals(Arrays.asList("foo"), ContainerBuilder.safeStringSplit("foo ,", ",")); - assertEquals(Arrays.asList("foo", "bar"), ContainerBuilder.safeStringSplit("foo, bar", ",")); + assertEquals(List.of("foo"), ContainerBuilder.safeStringSplit("foo", ",")); + assertEquals(List.of("foo"), ContainerBuilder.safeStringSplit(" foo", ",")); + assertEquals(List.of("foo"), ContainerBuilder.safeStringSplit("foo ", ",")); + assertEquals(List.of("foo"), ContainerBuilder.safeStringSplit("foo, ", ",")); + assertEquals(List.of("foo"), ContainerBuilder.safeStringSplit("foo ,", ",")); + assertEquals(List.of("foo", "bar"), ContainerBuilder.safeStringSplit("foo, bar", ",")); } } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/GlobPatternTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/GlobPatternTestCase.java index ef09c854f72..0e5df5656c4 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/GlobPatternTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/GlobPatternTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.jdisc.application; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -42,14 +41,14 @@ public class GlobPatternTestCase { assertMatch("foo", "foo", Collections.<String>emptyList()); assertNotMatch("foo", "bar"); - assertMatch("*", "foo", Arrays.asList("foo")); - assertMatch("*", "bar", Arrays.asList("bar")); + assertMatch("*", "foo", List.of("foo")); + assertMatch("*", "bar", List.of("bar")); - assertMatch("*foo", "foo", Arrays.asList("")); - assertMatch("*oo", "foo", Arrays.asList("f")); - assertMatch("f*o", "foo", Arrays.asList("o")); - assertMatch("fo*", "foo", Arrays.asList("o")); - assertMatch("foo*", "foo", Arrays.asList("")); + assertMatch("*foo", "foo", List.of("")); + assertMatch("*oo", "foo", List.of("f")); + assertMatch("f*o", "foo", List.of("o")); + assertMatch("fo*", "foo", List.of("o")); + assertMatch("foo*", "foo", List.of("")); assertNotMatch("*foo", "bar"); assertNotMatch("*oo", "bar"); @@ -57,11 +56,11 @@ public class GlobPatternTestCase { assertNotMatch("fo*", "bar"); assertNotMatch("foo*", "bar"); - assertMatch("**foo", "foo", Arrays.asList("", "")); - assertMatch("**oo", "foo", Arrays.asList("", "f")); - assertMatch("f**o", "foo", Arrays.asList("", "o")); - assertMatch("fo**", "foo", Arrays.asList("", "o")); - assertMatch("foo**", "foo", Arrays.asList("", "")); + assertMatch("**foo", "foo", List.of("", "")); + assertMatch("**oo", "foo", List.of("", "f")); + assertMatch("f**o", "foo", List.of("", "o")); + assertMatch("fo**", "foo", List.of("", "o")); + assertMatch("foo**", "foo", List.of("", "")); assertNotMatch("**foo", "bar"); assertNotMatch("**oo", "bar"); @@ -70,9 +69,9 @@ public class GlobPatternTestCase { assertNotMatch("foo**", "bar"); assertMatch("foo bar", "foo bar", Collections.<String>emptyList()); - assertMatch("*foo *bar", "foo bar", Arrays.asList("", "")); - assertMatch("foo* bar*", "foo bar", Arrays.asList("", "")); - assertMatch("f* *r", "foo bar", Arrays.asList("oo", "ba")); + assertMatch("*foo *bar", "foo bar", List.of("", "")); + assertMatch("foo* bar*", "foo bar", List.of("", "")); + assertMatch("f* *r", "foo bar", List.of("oo", "ba")); assertNotMatch("foo bar", "baz cox"); assertNotMatch("*foo *bar", "baz cox"); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryTestCase.java index 7aa8a16d856..30c917f9150 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryTestCase.java @@ -12,7 +12,6 @@ import com.google.inject.name.Named; import com.google.inject.name.Names; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -47,7 +46,7 @@ public class GuiceRepositoryTestCase { GuiceRepository guice = new GuiceRepository(); StringBinding foo = new StringBinding("fooKey", "fooVal"); StringBinding bar = new StringBinding("barKey", "barVal"); - guice.installAll(Arrays.asList(foo, bar)); + guice.installAll(List.of(foo, bar)); assertBinding(guice, "fooKey", "fooVal"); assertBinding(guice, "barKey", "barVal"); @@ -77,12 +76,12 @@ public class GuiceRepositoryTestCase { StringBinding foo = new StringBinding("fooKey", "fooVal"); StringBinding bar = new StringBinding("barKey", "barVal"); StringBinding baz = new StringBinding("bazKey", "bazVal"); - guice.installAll(Arrays.asList(foo, bar, baz)); + guice.installAll(List.of(foo, bar, baz)); assertBinding(guice, "fooKey", "fooVal"); assertBinding(guice, "barKey", "barVal"); assertBinding(guice, "bazKey", "bazVal"); - guice.uninstallAll(Arrays.asList(foo, baz)); + guice.uninstallAll(List.of(foo, baz)); assertNoBinding(guice, "fooKey"); assertBinding(guice, "barKey", "barVal"); assertNoBinding(guice, "bazKey"); @@ -122,7 +121,7 @@ public class GuiceRepositoryTestCase { void requireThatPrivateModulesWorks() { GuiceRepository guice = new GuiceRepository(); - List<Named> names = Arrays.asList(Names.named("A"), Names.named("B")); + List<Named> names = List.of(Names.named("A"), Names.named("B")); for (Named name : names) { guice.install(createPrivateInjectNameModule(name)); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/ServerRepositoryTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/ServerRepositoryTestCase.java index 3a2db7d1e75..a54a42228e4 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/ServerRepositoryTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/ServerRepositoryTestCase.java @@ -5,8 +5,8 @@ import com.yahoo.jdisc.NoopSharedResource; import com.yahoo.jdisc.service.ServerProvider; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -35,7 +35,7 @@ public class ServerRepositoryTestCase { ServerRepository servers = newServerRepository(); ServerProvider foo = new MyServer(); ServerProvider bar = new MyServer(); - servers.installAll(Arrays.asList(foo, bar)); + servers.installAll(List.of(foo, bar)); Iterator<ServerProvider> it = servers.iterator(); assertTrue(it.hasNext()); @@ -60,8 +60,8 @@ public class ServerRepositoryTestCase { ServerProvider foo = new MyServer(); ServerProvider bar = new MyServer(); ServerProvider baz = new MyServer(); - servers.installAll(Arrays.asList(foo, bar, baz)); - servers.uninstallAll(Arrays.asList(foo, bar)); + servers.installAll(List.of(foo, bar, baz)); + servers.uninstallAll(List.of(foo, bar)); Iterator<ServerProvider> it = servers.iterator(); assertNotNull(it); assertTrue(it.hasNext()); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java index fa949ccaabd..0f8016eba10 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/application/UriPatternTestCase.java @@ -5,8 +5,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -20,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.fail; */ public class UriPatternTestCase { - private static final List<String> NO_GROUPS = Collections.emptyList(); + private static final List<String> NO_GROUPS = List.of(); @Test void requireThatIllegalPatternsAreDetected() { @@ -50,19 +48,19 @@ public class UriPatternTestCase { assertNotMatch(pattern, "barbaz://host:69/path"); pattern = new UriPattern("*://host:69/path"); - assertMatch(pattern, "foobar://host:69/path", Arrays.asList("foobar")); - assertMatch(pattern, "bar://host:69/path", Arrays.asList("bar")); - assertMatch(pattern, "barbaz://host:69/path", Arrays.asList("barbaz")); + assertMatch(pattern, "foobar://host:69/path", List.of("foobar")); + assertMatch(pattern, "bar://host:69/path", List.of("bar")); + assertMatch(pattern, "barbaz://host:69/path", List.of("barbaz")); pattern = new UriPattern("*bar://host:69/path"); - assertMatch(pattern, "foobar://host:69/path", Arrays.asList("foo")); - assertMatch(pattern, "bar://host:69/path", Arrays.asList("")); + assertMatch(pattern, "foobar://host:69/path", List.of("foo")); + assertMatch(pattern, "bar://host:69/path", List.of("")); assertNotMatch(pattern, "barbaz://host:69/path"); pattern = new UriPattern("bar*://host:69/path"); assertNotMatch(pattern, "foobar://host:69/path"); - assertMatch(pattern, "bar://host:69/path", Arrays.asList("")); - assertMatch(pattern, "barbaz://host:69/path", Arrays.asList("baz")); + assertMatch(pattern, "bar://host:69/path", List.of("")); + assertMatch(pattern, "barbaz://host:69/path", List.of("baz")); // host matching pattern = new UriPattern("scheme://bar:69/path"); @@ -71,19 +69,19 @@ public class UriPatternTestCase { assertNotMatch(pattern, "scheme://barbaz:69/path"); pattern = new UriPattern("scheme://*:69/path"); - assertMatch(pattern, "scheme://foobar:69/path", Arrays.asList("foobar")); - assertMatch(pattern, "scheme://bar:69/path", Arrays.asList("bar")); - assertMatch(pattern, "scheme://barbaz:69/path", Arrays.asList("barbaz")); + assertMatch(pattern, "scheme://foobar:69/path", List.of("foobar")); + assertMatch(pattern, "scheme://bar:69/path", List.of("bar")); + assertMatch(pattern, "scheme://barbaz:69/path", List.of("barbaz")); pattern = new UriPattern("scheme://*bar:69/path"); - assertMatch(pattern, "scheme://foobar:69/path", Arrays.asList("foo")); - assertMatch(pattern, "scheme://bar:69/path", Arrays.asList("")); + assertMatch(pattern, "scheme://foobar:69/path", List.of("foo")); + assertMatch(pattern, "scheme://bar:69/path", List.of("")); assertNotMatch(pattern, "scheme://barbaz:69/path"); pattern = new UriPattern("scheme://bar*:69/path"); assertNotMatch(pattern, "scheme://foobar:69/path"); - assertMatch(pattern, "scheme://bar:69/path", Arrays.asList("")); - assertMatch(pattern, "scheme://barbaz:69/path", Arrays.asList("baz")); + assertMatch(pattern, "scheme://bar:69/path", List.of("")); + assertMatch(pattern, "scheme://barbaz:69/path", List.of("baz")); // port matching pattern = new UriPattern("scheme://host:69/path"); @@ -92,9 +90,9 @@ public class UriPatternTestCase { assertNotMatch(pattern, "scheme://host:699/path"); pattern = new UriPattern("scheme://host:*/path"); - assertMatch(pattern, "scheme://host:669/path", Arrays.asList("669")); - assertMatch(pattern, "scheme://host:69/path", Arrays.asList("69")); - assertMatch(pattern, "scheme://host:699/path", Arrays.asList("699")); + assertMatch(pattern, "scheme://host:669/path", List.of("669")); + assertMatch(pattern, "scheme://host:69/path", List.of("69")); + assertMatch(pattern, "scheme://host:699/path", List.of("699")); // path matching pattern = new UriPattern("scheme://host:69/"); @@ -107,20 +105,20 @@ public class UriPatternTestCase { assertNotMatch(pattern, "scheme://host:69/barbaz"); pattern = new UriPattern("scheme://host:69/*"); - assertMatch(pattern, "scheme://host:69/", Arrays.asList("")); - assertMatch(pattern, "scheme://host:69/foobar", Arrays.asList("foobar")); - assertMatch(pattern, "scheme://host:69/bar", Arrays.asList("bar")); - assertMatch(pattern, "scheme://host:69/barbaz", Arrays.asList("barbaz")); + assertMatch(pattern, "scheme://host:69/", List.of("")); + assertMatch(pattern, "scheme://host:69/foobar", List.of("foobar")); + assertMatch(pattern, "scheme://host:69/bar", List.of("bar")); + assertMatch(pattern, "scheme://host:69/barbaz", List.of("barbaz")); pattern = new UriPattern("scheme://host:69/*bar"); - assertMatch(pattern, "scheme://host:69/foobar", Arrays.asList("foo")); - assertMatch(pattern, "scheme://host:69/bar", Arrays.asList("")); + assertMatch(pattern, "scheme://host:69/foobar", List.of("foo")); + assertMatch(pattern, "scheme://host:69/bar", List.of("")); assertNotMatch(pattern, "scheme://host:69/barbaz"); pattern = new UriPattern("scheme://host:69/bar*"); assertNotMatch(pattern, "scheme://host:69/foobar"); - assertMatch(pattern, "scheme://host:69/bar", Arrays.asList("")); - assertMatch(pattern, "scheme://host:69/barbaz", Arrays.asList("baz")); + assertMatch(pattern, "scheme://host:69/bar", List.of("")); + assertMatch(pattern, "scheme://host:69/barbaz", List.of("baz")); } @Test @@ -129,7 +127,7 @@ public class UriPatternTestCase { assertNotMatch(pattern, "scheme://host"); pattern = new UriPattern("scheme://host/*"); - assertMatch(pattern, "scheme://host", Arrays.asList("")); + assertMatch(pattern, "scheme://host", List.of("")); } @Test @@ -142,8 +140,8 @@ public class UriPatternTestCase { @Test void requireThatHostSupportsWildcard() { UriPattern pattern = new UriPattern("scheme://*.host/path"); - assertMatch(pattern, "scheme://a.host/path", Arrays.asList("a")); - assertMatch(pattern, "scheme://a.b.host/path", Arrays.asList("a.b")); + assertMatch(pattern, "scheme://a.host/path", List.of("a")); + assertMatch(pattern, "scheme://a.b.host/path", List.of("a.b")); } @Test diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/BindingMatchingTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/BindingMatchingTestCase.java index 478f15f721d..53a29cf22c1 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/BindingMatchingTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/BindingMatchingTestCase.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -36,7 +35,7 @@ public class BindingMatchingTestCase { void runThroughtputMeasurements() throws Exception { System.err.format("%15s%15s%15s%15s%15s%15s%15s%15s\n", "No. of Bindings", "1 thread", "2 thread", "4 thread", "8 thread", "16 thread", "32 thread", "64 thread"); - for (int numBindings : Arrays.asList(1, 10, 25, 50, 100, 250)) { + for (int numBindings : List.of(1, 10, 25, 50, 100, 250)) { BindingRepository<Object> repo = new BindingRepository<>(); for (int binding = 0; binding < numBindings; ++binding) { repo.bind("http://*/v" + binding + "/*/data/", new Object()); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/UriMatchingTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/UriMatchingTestCase.java index 5301eb297b4..cb9114f2981 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/UriMatchingTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/benchmark/UriMatchingTestCase.java @@ -6,7 +6,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -20,7 +19,7 @@ public class UriMatchingTestCase { @Test void requireThatUriPatternMatchingIsFast() { - List<String> inputs = Arrays.asList( + List<String> inputs = List.of( "other://host/", "scheme://other/", "scheme://host/", diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationEnvironmentModuleTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationEnvironmentModuleTestCase.java index dfc52b92583..af3fa418556 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationEnvironmentModuleTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationEnvironmentModuleTestCase.java @@ -17,7 +17,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ThreadFactory; -import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -40,7 +39,7 @@ public class ApplicationEnvironmentModuleTestCase { expected.add(entry.getKey().getTypeLiteral().getRawType()); } - ApplicationLoader loader = new ApplicationLoader(new NonWorkingOsgiFramework(), emptyList()); + ApplicationLoader loader = new ApplicationLoader(new NonWorkingOsgiFramework(), List.of()); injector = Guice.createInjector(new ApplicationEnvironmentModule(loader)); for (Map.Entry<Key<?>, Binding<?>> entry : injector.getBindings().entrySet()) { assertNotNull(expected.remove(entry.getKey().getTypeLiteral().getRawType())); @@ -50,7 +49,7 @@ public class ApplicationEnvironmentModuleTestCase { @Test void requireThatContainerBuilderCanBeInjected() { - ApplicationLoader loader = new ApplicationLoader(new NonWorkingOsgiFramework(), emptyList()); + ApplicationLoader loader = new ApplicationLoader(new NonWorkingOsgiFramework(), List.of()); assertNotNull(new ApplicationEnvironmentModule(loader).containerBuilder()); assertNotNull(Guice.createInjector(new ApplicationEnvironmentModule(loader)) .getInstance(ContainerBuilder.class)); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationRestartTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationRestartTestCase.java index d8aa553b628..95511b3ed1c 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationRestartTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ApplicationRestartTestCase.java @@ -15,7 +15,7 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.nio.ByteBuffer; -import java.util.Arrays; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -54,7 +54,7 @@ public class ApplicationRestartTestCase { private static ApplicationLoader newApplicationLoader() { return new ApplicationLoader(new NonWorkingOsgiFramework(), - Arrays.asList(new AbstractModule() { + List.of(new AbstractModule() { @Override public void configure() { bind(Application.class).to(SimpleApplication.class); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java index 252b7074a2c..25399433e2e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.jdisc.core; import org.junit.jupiter.api.Test; import org.osgi.framework.Constants; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -60,8 +59,8 @@ public class FelixParamsTestCase { assertEquals(prev.length + 2, next.length); List<String> diff = new LinkedList<>(); - diff.addAll(Arrays.asList(next)); - diff.removeAll(Arrays.asList(prev)); + diff.addAll(List.of(next)); + diff.removeAll(List.of(prev)); assertEquals(2, diff.size()); assertTrue(diff.contains("foo")); assertTrue(diff.contains("bar")); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java index a9ac904c36c..c37b6ffe915 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java @@ -3,8 +3,8 @@ package com.yahoo.jdisc.core; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.LinkedList; +import java.util.List; import java.util.Queue; import static com.yahoo.jdisc.core.ScheduledQueue.MILLIS_PER_SLOT; @@ -123,6 +123,6 @@ public class ScheduledQueueTestCase { Queue<Object> expired = new LinkedList<>(); queue.drainTo(currentTimeMillis, expired); assertEquals(expected.length, expired.size()); - assertEquals(Arrays.asList(expected), expired); + assertEquals(List.of(expected), expired); } } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/RequestDispatchTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/RequestDispatchTestCase.java index c672b33ff2b..acfe8d32e61 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/RequestDispatchTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/RequestDispatchTestCase.java @@ -12,7 +12,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -31,7 +30,7 @@ public class RequestDispatchTestCase { @Test void requireThatRequestCanBeDispatched() throws Exception { final TestDriver driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(); - final List<ByteBuffer> writtenContent = Arrays.asList(ByteBuffer.allocate(6), ByteBuffer.allocate(9)); + final List<ByteBuffer> writtenContent = List.of(ByteBuffer.allocate(6), ByteBuffer.allocate(9)); ReadableContentChannel receivedContent = new ReadableContentChannel(); ContainerBuilder builder = driver.newContainerBuilder(); Response response = new Response(Response.Status.OK); @@ -154,7 +153,7 @@ public class RequestDispatchTestCase { @Override protected Iterable<ByteBuffer> requestContent() { - return Arrays.asList(ByteBuffer.allocate(69)); + return List.of(ByteBuffer.allocate(69)); } }.dispatch(); fail(); @@ -192,7 +191,7 @@ public class RequestDispatchTestCase { @Override protected Iterable<ByteBuffer> requestContent() { - return Arrays.asList(ByteBuffer.allocate(69)); + return List.of(ByteBuffer.allocate(69)); } }.dispatch(); fail(); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ResponseDispatchTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ResponseDispatchTestCase.java index 84bdc07dd33..32481b4607e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ResponseDispatchTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ResponseDispatchTestCase.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,7 +45,7 @@ public class ResponseDispatchTestCase { ReadableContentChannel content = new ReadableContentChannel(); FutureResponse handler = new FutureResponse(content); ByteBuffer buf = ByteBuffer.allocate(69); - ResponseDispatch.newInstance(69, Arrays.asList(buf)).dispatch(handler); + ResponseDispatch.newInstance(69, List.of(buf)).dispatch(handler); Response response = handler.get(600, TimeUnit.SECONDS); assertNotNull(response); assertEquals(69, response.getStatus()); @@ -57,7 +56,7 @@ public class ResponseDispatchTestCase { ReadableContentChannel content = new ReadableContentChannel(); FutureResponse handler = new FutureResponse(content); ByteBuffer buf = ByteBuffer.allocate(69); - ResponseDispatch.newInstance(69, Arrays.asList(buf)).dispatch(handler); + ResponseDispatch.newInstance(69, List.of(buf)).dispatch(handler); Response response = handler.get(600, TimeUnit.SECONDS); assertNotNull(response); assertEquals(69, response.getStatus()); @@ -69,7 +68,7 @@ public class ResponseDispatchTestCase { FutureResponse handler = new FutureResponse(content); ByteBuffer buf = ByteBuffer.allocate(69); Response sentResponse = new Response(69); - ResponseDispatch.newInstance(sentResponse, Arrays.asList(buf)).dispatch(handler); + ResponseDispatch.newInstance(sentResponse, List.of(buf)).dispatch(handler); Response receivedResponse = handler.get(600, TimeUnit.SECONDS); assertSame(sentResponse, receivedResponse); assertSame(buf, content.read()); @@ -80,7 +79,7 @@ public class ResponseDispatchTestCase { @Test void requireThatResponseCanBeDispatched() throws Exception { final Response response = new Response(Response.Status.OK); - final List<ByteBuffer> writtenContent = Arrays.asList(ByteBuffer.allocate(6), ByteBuffer.allocate(9)); + final List<ByteBuffer> writtenContent = List.of(ByteBuffer.allocate(6), ByteBuffer.allocate(9)); ResponseDispatch dispatch = new ResponseDispatch() { @Override diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ThreadedRequestHandlerTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ThreadedRequestHandlerTestCase.java index 84da19c221a..56d7d2959e8 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ThreadedRequestHandlerTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/handler/ThreadedRequestHandlerTestCase.java @@ -10,7 +10,7 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.nio.ByteBuffer; -import java.util.Arrays; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; @@ -170,7 +170,7 @@ public class ThreadedRequestHandlerTestCase { @Override protected Iterable<ByteBuffer> requestContent() { - return Arrays.asList(content); + return List.of(content); } }.dispatch(); } diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryIntegrationTest.java index f3d936b20d0..9c37fc7e0de 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/GuiceRepositoryIntegrationTest.java @@ -10,7 +10,6 @@ import org.junit.Test; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -56,7 +55,7 @@ public class GuiceRepositoryIntegrationTest { BundleInstaller installer = new BundleInstaller(driver.osgiFramework()); Bundle bundle = installer.installAndStart("my-guice-module.jar").get(0); - builder.guiceModules().installAll(bundle, Arrays.asList("com.yahoo.jdisc.bundle.MyGuiceModule", + builder.guiceModules().installAll(bundle, List.of("com.yahoo.jdisc.bundle.MyGuiceModule", "com.yahoo.jdisc.bundle.MyGuiceModule")); List<Module> next = new LinkedList<>(builder.guiceModules().collection()); next.removeAll(prev); diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/ServerRepositoryIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/ServerRepositoryIntegrationTest.java index 04b88654112..1232e0ecf0d 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/ServerRepositoryIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/ServerRepositoryIntegrationTest.java @@ -8,8 +8,8 @@ import com.yahoo.jdisc.test.TestDriver; import org.junit.Test; import org.osgi.framework.Bundle; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -54,7 +54,7 @@ public class ServerRepositoryIntegrationTest { BundleInstaller installer = new BundleInstaller(driver.osgiFramework()); Bundle bundle = installer.installAndStart("my-server-provider.jar").get(0); ContainerBuilder builder = driver.newContainerBuilder(); - builder.serverProviders().installAll(bundle, Arrays.asList("com.yahoo.jdisc.bundle.MyServerProvider", + builder.serverProviders().installAll(bundle, List.of("com.yahoo.jdisc.bundle.MyServerProvider", "com.yahoo.jdisc.bundle.MyServerProvider")); Iterator<ServerProvider> it = builder.serverProviders().iterator(); assertTrue(it.hasNext()); diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ApplicationLoaderIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ApplicationLoaderIntegrationTest.java index a0fde346901..cf8914ea39a 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ApplicationLoaderIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ApplicationLoaderIntegrationTest.java @@ -8,8 +8,8 @@ import org.apache.felix.framework.util.FelixConstants; import org.junit.Test; import org.osgi.framework.Bundle; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -29,7 +29,7 @@ public class ApplicationLoaderIntegrationTest { public void requireThatLifecycleWorks() throws Exception { MyModule module = new MyModule(); ApplicationLoader loader = new ApplicationLoader(TestDriver.newOsgiFramework(), - Arrays.asList(module)); + List.of(module)); loader.init("app-a.jar", false); assertFalse(module.init.await(100, TimeUnit.MILLISECONDS)); diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/FelixFrameworkIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/FelixFrameworkIntegrationTest.java index f91407326c5..53a2b20a0b1 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/FelixFrameworkIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/FelixFrameworkIntegrationTest.java @@ -8,7 +8,6 @@ import org.junit.Test; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; -import java.util.Arrays; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -214,7 +213,7 @@ public class FelixFrameworkIntegrationTest { for (Bundle bundle : bundles) { actual.add(bundle.getSymbolicName()); } - assertEquals(Arrays.asList(expectedBundles), actual); + assertEquals(List.of(expectedBundles), actual); felix.startBundles(bundles, false); if (className != null) { assertNotNull(bundles.get(0).loadClass(className).getDeclaredConstructor().newInstance()); diff --git a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java index 70beb291527..0de39cbbcd2 100644 --- a/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java +++ b/jrt/src/com/yahoo/jrt/tool/RpcInvoker.java @@ -17,7 +17,6 @@ import com.yahoo.jrt.Value; import com.yahoo.jrt.Values; import java.time.Duration; -import java.util.Arrays; import java.util.List; import java.util.ArrayList; @@ -108,7 +107,7 @@ public class RpcInvoker { System.err.println(" supported types: {'b','h','i','l','f','d','s'}"); System.exit(0); } - List<String> arguments = new ArrayList<String>(Arrays.asList(args)); + List<String> arguments = new ArrayList<String>(List.of(args)); String connectSpec = "localhost:8086"; if ("-h".equals(arguments.get(0)) && arguments.size() >= 3) { arguments.remove(0); // Consume -h diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java index 7bad0e64aa8..772e7722391 100644 --- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java +++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java @@ -25,7 +25,6 @@ import static com.yahoo.security.X509CertificateBuilder.generateRandomSerialNumb import static java.time.Instant.EPOCH; import static java.time.temporal.ChronoUnit.DAYS; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; /** * @author bjorncs @@ -42,8 +41,7 @@ class CryptoUtils { singleton( new PeerPolicy( "localhost-policy", - singletonList( - RequiredPeerCredential.of(Field.CN, "localhost"))))); + List.of(RequiredPeerCredential.of(Field.CN, "localhost"))))); static TlsContext createTestTlsContext() { return DefaultTlsContext.of( diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index 44ed57834d1..1ca32a2dd37 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -346,8 +346,9 @@ "public com.yahoo.language.process.Embedder$Context setDestination(java.lang.String)", "public java.lang.String getEmbedderId()", "public com.yahoo.language.process.Embedder$Context setEmbedderId(java.lang.String)", - "public void putCachedValue(java.lang.String, java.lang.Object)", - "public java.lang.Object getCachedValue(java.lang.String)" + "public void putCachedValue(java.lang.Object, java.lang.Object)", + "public java.lang.Object getCachedValue(java.lang.Object)", + "public java.lang.Object computeCachedValueIfAbsent(java.lang.Object, java.util.function.Supplier)" ], "fields" : [ ] }, diff --git a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java index 2ab2de303c2..989edcdb18a 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java @@ -7,10 +7,10 @@ import com.yahoo.language.Language; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * An embedder converts a text string to a tensor @@ -73,9 +73,10 @@ public interface Embedder { */ @Beta interface Runtime { - /** Sample latency metric for embedding */ + + /** Add a sample embedding latency to this */ void sampleEmbeddingLatency(double millis, Context ctx); - /** Sample sequence length metric for embedding */ + /** Add a sample embedding length to this */ void sampleSequenceLength(long length, Context ctx); static Runtime testInstance() { @@ -91,7 +92,7 @@ public interface Embedder { private Language language = Language.UNKNOWN; private String destination; private String embedderId = "unknown"; - private final Map<String, Object> cache; + private final Map<Object, Object> cache; public Context(String destination) { this(destination, LazyMap.newHashMap()); @@ -101,7 +102,7 @@ public interface Embedder { * @param destination the name of the recipient of this tensor * @param cache a cache shared between all embed invocations for a single request */ - public Context(String destination, Map<String, Object> cache) { + public Context(String destination, Map<Object, Object> cache) { this.destination = destination; this.cache = Objects.requireNonNull(cache); } @@ -153,15 +154,21 @@ public interface Embedder { return this; } - public void putCachedValue(String key, Object value) { + public void putCachedValue(Object key, Object value) { cache.put(key, value); } /** Returns a cached value, or null if not present. */ - public Object getCachedValue(String key) { + public Object getCachedValue(Object key) { return cache.get(key); } + /** Returns the cached value, or computes and caches it if not present. */ + @SuppressWarnings("unchecked") + public <T> T computeCachedValueIfAbsent(Object key, Supplier<? extends T> supplier) { + return (T) cache.computeIfAbsent(key, __ -> supplier.get()); + } + } class FailingEmbedder implements Embedder { diff --git a/linguistics/src/test/java/com/yahoo/language/LanguageTestCase.java b/linguistics/src/test/java/com/yahoo/language/LanguageTestCase.java index 1b1c9eb21cb..3be57cb21cf 100644 --- a/linguistics/src/test/java/com/yahoo/language/LanguageTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/LanguageTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.language; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.*; @@ -15,7 +14,7 @@ public class LanguageTestCase { @Test public void requireThatSpecificLanguagesAreCjk() { - List<Language> cjk = Arrays.asList(Language.CHINESE_SIMPLIFIED, + List<Language> cjk = List.of(Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, Language.JAPANESE, Language.KOREAN, diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index b6e33d70ae6..69094cab8f2 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -6,8 +6,8 @@ import com.yahoo.language.process.GramSplitter.GramSplitterIterator; import com.yahoo.language.simple.SimpleLinguistics; import org.junit.Test; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -220,7 +220,7 @@ public class GramSplitterTestCase { } private void assertGramSplits(String input, int gramSize, String ... expected) { - assertEquals(Arrays.asList(expected), gramSplitter.split(input, gramSize).toExtractedList()); + assertEquals(List.of(expected), gramSplitter.split(input, gramSize).toExtractedList()); } private void assertGramSplit(String input, int gramSize, String expected) { diff --git a/linguistics/src/test/java/com/yahoo/language/process/SegmenterImplTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SegmenterImplTestCase.java index f4ec53d1f38..7e097736abe 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/SegmenterImplTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/SegmenterImplTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.language.simple.SimpleTokenizer; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -22,25 +21,25 @@ public class SegmenterImplTestCase { @Test public void requireThatNonIndexableCharactersAreDelimiters() { - assertSegments("i've", Arrays.asList("i", "ve")); - assertSegments("foo bar. baz", Arrays.asList("foo", "bar", "baz")); - assertSegments("1,2, 3 4", Arrays.asList("1", "2", "3", "4")); + assertSegments("i've", List.of("i", "ve")); + assertSegments("foo bar. baz", List.of("foo", "bar", "baz")); + assertSegments("1,2, 3 4", List.of("1", "2", "3", "4")); } @Test public void requireThatAdjacentIndexableTokenTypesAreNotSplit() { - assertSegments("a1,2b,c3,4d", Arrays.asList("a1", "2b", "c3", "4d")); + assertSegments("a1,2b,c3,4d", List.of("a1", "2b", "c3", "4d")); } @Test public void requireThatSegmentationReturnsOriginalForm() { - assertSegments("a\u030A", Arrays.asList("a\u030A")); - assertSegments("FOO BAR", Arrays.asList("FOO", "BAR")); + assertSegments("a\u030A", List.of("a\u030A")); + assertSegments("FOO BAR", List.of("FOO", "BAR")); } @Test public void requireThatEmptyInputIsPreserved() { - assertSegments("", Arrays.asList("")); + assertSegments("", List.of("")); } private static void assertSegments(String input, List<String> expectedSegments) { @@ -51,7 +50,7 @@ public class SegmenterImplTestCase { public void requireThatEmptyStringsAreSuppressed() { Tokenizer fancyTokenizer = new FancyTokenizer(); Segmenter fancySegmenter = new SegmenterImpl(fancyTokenizer); - List<String> expectedSegments = Arrays.asList("juice", "\u00BD", "oz"); + List<String> expectedSegments = List.of("juice", "\u00BD", "oz"); String input = "juice \u00BD oz"; assertEquals(expectedSegments, fancySegmenter.segment(input, Language.ENGLISH)); } diff --git a/linguistics/src/test/java/com/yahoo/language/process/StemmerImplTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/StemmerImplTestCase.java index 9c5914baeb9..785225a5096 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/StemmerImplTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/StemmerImplTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.language.simple.SimpleTokenizer; import org.junit.Test; import org.mockito.Mockito; -import java.util.Arrays; import java.util.List; import java.util.ArrayList; @@ -21,13 +20,13 @@ public class StemmerImplTestCase { @Test public void requireThatStemIsNormalizedAndLowerCased() { - assertStem("FOO", Arrays.asList("foo")); - assertStem("a\u030A", Arrays.asList("\u00E5")); + assertStem("FOO", List.of("foo")); + assertStem("a\u030A", List.of("\u00E5")); } @Test public void requireThatOnlyIndexableTokensAreReturned() { - assertStem("foo. (bar)!", Arrays.asList("foo", "bar")); + assertStem("foo. (bar)!", List.of("foo", "bar")); } @Test @@ -43,17 +42,17 @@ public class StemmerImplTestCase { Tokenizer tokenizer = Mockito.mock(Tokenizer.class); Mockito.when(tokenizer.tokenize(Mockito.anyString(), Mockito.<Language>any(), Mockito.<StemMode>any(), Mockito.anyBoolean())) - .thenReturn(Arrays.<Token>asList(token)); + .thenReturn(List.of(token)); Stemmer stemmer = new StemmerImpl(tokenizer); token.setSpecialToken(false); - assertEquals(Arrays.asList(new StemList("c"), + assertEquals(List.of(new StemList("c"), new StemList("p"), new StemList("p")), stemmer.stem("c++", StemMode.SHORTEST, Language.ENGLISH)); token.setSpecialToken(true); - assertEquals(Arrays.asList(new StemList("c++")), + assertEquals(List.of(new StemList("c++")), stemmer.stem("c++", StemMode.SHORTEST, Language.ENGLISH)); } diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java index cf5a26c1f04..2cfb6c33b93 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java @@ -7,7 +7,6 @@ import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -31,23 +30,23 @@ public class TokenizationTestCase { @Test public void testTokenizer() { assertTokenize("This is a test, 123", - Arrays.asList("this", "is", "a", "test", "123"), - Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); + List.of("this", "is", "a", "test", "123"), + List.of("This", " ", "is", " ", "a", " ", "test", ",", " ", "123")); } @Test public void testUnderScoreTokenization() { - assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null); + assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, List.of("ugcapi", "1"), null); } @Test public void testPhrasesWithPunctuation() { assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE, false, - Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), + List.of("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"), null); - assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null); - assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null); + assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, List.of("phy", "101"), null); + assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, List.of("101", "3"), null); } @Test diff --git a/linguistics/src/test/java/com/yahoo/language/simple/TokenizerTester.java b/linguistics/src/test/java/com/yahoo/language/simple/TokenizerTester.java index 401b89f0696..021f04b8b87 100644 --- a/linguistics/src/test/java/com/yahoo/language/simple/TokenizerTester.java +++ b/linguistics/src/test/java/com/yahoo/language/simple/TokenizerTester.java @@ -8,7 +8,6 @@ import com.yahoo.language.process.Token; import com.yahoo.language.process.TokenScript; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -28,7 +27,7 @@ public class TokenizerTester { for (Token token : tokenize(input)) { findTokenStrings(token, actual); } - assertEquals(Arrays.asList(expectedTokenStrings), actual); + assertEquals(List.of(expectedTokenStrings), actual); } public void assertTokenScripts(String input, TokenScript... expectedTokenScripts) { @@ -36,7 +35,7 @@ public class TokenizerTester { for (Token token : tokenize(input)) { findTokenScripts(token, actual); } - assertEquals(Arrays.asList(expectedTokenScripts), actual); + assertEquals(List.of(expectedTokenScripts), actual); } public List<String> findTokenStrings(Token token, List<String> out) { diff --git a/logforwarder/CMakeLists.txt b/logforwarder/CMakeLists.txt index 85406cba5de..b8f7fb3c416 100644 --- a/logforwarder/CMakeLists.txt +++ b/logforwarder/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_define_module( APPS src/apps/vespa-logforwarder-start + src/apps/vespa-otelcol-start ) diff --git a/logforwarder/src/apps/vespa-otelcol-start/.gitignore b/logforwarder/src/apps/vespa-otelcol-start/.gitignore new file mode 100644 index 00000000000..f2e64aa95d6 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/.gitignore @@ -0,0 +1 @@ +vespa-otelcol-start diff --git a/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt b/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt new file mode 100644 index 00000000000..d95ce0584c9 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(otelcol_start_app + SOURCES + main.cpp + cf-handler.cpp + child-handler.cpp + file-watcher.cpp + wrapper.cpp + OUTPUT_NAME vespa-otelcol-start + INSTALL bin + DEPENDS + config_cloudconfig + configdefinitions + vespalib +) diff --git a/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp new file mode 100644 index 00000000000..9579271f88a --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.cpp @@ -0,0 +1,67 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cf-handler.h" +#include <vespa/config/common/configcontext.h> +#include <vespa/config/common/configsystem.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/legacy.h> +#include <vespa/config/subscription/configsubscriber.hpp> + +#include <unistd.h> + +#include <vespa/log/log.h> +LOG_SETUP(".cf-handler"); + +CfHandler::CfHandler(const std::string & configId) + : _subscriber(std::make_shared<config::ConfigContext>(*config::legacyConfigId2Spec(configId))) +{} + +CfHandler::~CfHandler() = default; + +void CfHandler::subscribe(const std::string & configId, std::chrono::milliseconds timeout) { + LOG(info, "subscribe with config id: %s", configId.c_str()); + std::string cfgId(config::legacyConfigId2ConfigId(configId)); + _handle = _subscriber.subscribe<OpenTelemetryConfig>(cfgId, timeout); +} + +void CfHandler::doConfigure() { + auto curConfig = _handle->getConfig(); + if (_lastConfig && *curConfig == *_lastConfig) { + LOG(info, "same config as last"); + return; + } + LOG(info, "new config, trigger restart"); + _lastConfig = std::move(curConfig); + const OpenTelemetryConfig& config(*_lastConfig); + LOG(info, "watch %zu files", config.refPaths.size()); + _fileWatcher.init(config.refPaths); + gotConfig(config); +} + +void CfHandler::checkConfig() { + if (_subscriber.nextConfigNow()) { + doConfigure(); + } else if (_fileWatcher.anyChanged()) { + LOG(info, "watched file updated, trigger restart"); + const OpenTelemetryConfig& config(*_lastConfig); + gotConfig(config); + } +} + +constexpr std::chrono::milliseconds CONFIG_TIMEOUT_MS(30 * 1000); + +void CfHandler::start(const std::string &configId) { + LOG(debug, "Reading configuration with id '%s'", configId.c_str()); + try { + subscribe(configId, CONFIG_TIMEOUT_MS); + } catch (config::ConfigTimeoutException & ex) { + LOG(warning, "Timout getting config, please check your setup. Will exit and restart: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); + } catch (config::InvalidConfigException& ex) { + LOG(error, "Fatal: Invalid configuration, please check your setup: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); + } catch (config::ConfigRuntimeException& ex) { + LOG(error, "Fatal: Could not get config, please check your setup: %s", ex.getMessage().c_str()); + std::_Exit(EXIT_FAILURE); + } +} diff --git a/logforwarder/src/apps/vespa-otelcol-start/cf-handler.h b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.h new file mode 100644 index 00000000000..8f9fe590376 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/cf-handler.h @@ -0,0 +1,24 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "file-watcher.h" +#include <vespa/config-open-telemetry.h> +#include <vespa/config/subscription/configsubscriber.h> + +using cloud::config::OpenTelemetryConfig; + +class CfHandler { +private: + FileWatcher _fileWatcher; + config::ConfigSubscriber _subscriber; + config::ConfigHandle<OpenTelemetryConfig>::UP _handle = {}; + std::unique_ptr<OpenTelemetryConfig> _lastConfig = {}; + void subscribe(const std::string & configId, std::chrono::milliseconds timeout); + void doConfigure(); +public: + CfHandler(const std::string &configId); + virtual ~CfHandler(); + void start(const std::string &configId); + void checkConfig(); + virtual void gotConfig(const OpenTelemetryConfig&) = 0; +}; diff --git a/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp b/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp new file mode 100644 index 00000000000..46ac3bea60e --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/child-handler.cpp @@ -0,0 +1,88 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "child-handler.h" + +#include <stdio.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <signal.h> +#include <sys/wait.h> +#include <vector> +#include <string> +#include <cstdlib> + +#include <vespa/log/log.h> +LOG_SETUP(".child-handler"); + +ChildHandler::ChildHandler() : _childRunning(false), _childPid(0) {} + +ChildHandler::~ChildHandler() = default; + +bool ChildHandler::checkChild() { + if (! _childRunning) return true; + int waitStatus = 0; + int r = waitpid(_childPid, &waitStatus, WNOHANG); + if (r == 0) { + return false; + } + if (r < 0) { + perror("waitpid"); + // XXX how to handle? + return false; + } + _childRunning = false; + if (WIFEXITED(waitStatus) && WEXITSTATUS(waitStatus) == 0) { + // all OK + LOG(info, "child ran ok, exit status 0"); + } else if (WIFEXITED(waitStatus)) { + LOG(warning, "child failed (exit status %d)", WEXITSTATUS(waitStatus)); + } else if (WIFSIGNALED(waitStatus)) { + if (_terminating) { + LOG(info, "child terminated (using signal %d)", WTERMSIG(waitStatus)); + } else { + LOG(warning, "child failed (exit on signal %d)", WTERMSIG(waitStatus)); + } + } else { + LOG(warning, "child failed (abnormal exit status %d)", waitStatus); + } + return true; +} + +void ChildHandler::startChild(const std::string &progPath, const std::string &cfPath) { + _terminating = false; + LOG(info, "startChild '%s' '%s'", progPath.c_str(), cfPath.c_str()); + pid_t child = fork(); + if (child == -1) { + perror("fork()"); + return; + } + if (child == 0) { + std::string cfArg{"--config=file:" + cfPath}; + const char *cargv[] = { progPath.c_str(), cfArg.c_str(), nullptr }; + execv(progPath.c_str(), const_cast<char **>(cargv)); + // if execv fails: + perror(progPath.c_str()); + std::_Exit(1); + } + LOG(info, "child running with pid %d", (int)child); + _childRunning = true; + _childPid = child; +} + +void ChildHandler::stopChild() { + if (! _childRunning) return; + LOG(info, "stopChild"); + _terminating = true; + kill(_childPid, SIGTERM); + for (int retry = 0; retry < 10; ++retry) { + if (checkChild()) return; + usleep(12500 + retry * 20000); + } + kill(_childPid, SIGKILL); + for (int retry = 0; retry < 10; ++retry) { + if (checkChild()) return; + usleep(12500 + retry * 20000); + } + LOG(error, "Could not terminete child process %d", _childPid); +} diff --git a/logforwarder/src/apps/vespa-otelcol-start/child-handler.h b/logforwarder/src/apps/vespa-otelcol-start/child-handler.h new file mode 100644 index 00000000000..4fd72fc7682 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/child-handler.h @@ -0,0 +1,18 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "cf-handler.h" +#include <string> + +class ChildHandler { +private: + bool _childRunning = false; + bool _terminating = false; + int _childPid = 0; +public: + ChildHandler(); + ~ChildHandler(); + void startChild(const std::string &progPath, const std::string &cfFile); + void stopChild(); + bool checkChild(); +}; diff --git a/logforwarder/src/apps/vespa-otelcol-start/file-watcher.cpp b/logforwarder/src/apps/vespa-otelcol-start/file-watcher.cpp new file mode 100644 index 00000000000..f56a1d9d169 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/file-watcher.cpp @@ -0,0 +1,37 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "file-watcher.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +namespace { + +time_t lastModTime(const vespalib::string &fn) { + if (fn.empty()) return 0; + struct stat info; + if (stat(fn.c_str(), &info) != 0) return 0; + return info.st_mtime; +} + +} // namespace + +bool FileWatcher::anyChanged() { + bool result = false; + for (auto &entry : watchedFiles) { + time_t updated = lastModTime(entry.pathName); + if (updated != entry.seenModTime) { + result = true; + entry.seenModTime = updated; + } + } + return result; +} + +void FileWatcher::init(const config::StringVector &pathList) { + watchedFiles.clear(); + for (const auto& path : pathList) { + watchedFiles.emplace_back(path, lastModTime(path)); + } +} diff --git a/logforwarder/src/apps/vespa-otelcol-start/file-watcher.h b/logforwarder/src/apps/vespa-otelcol-start/file-watcher.h new file mode 100644 index 00000000000..0f50f6d90f7 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/file-watcher.h @@ -0,0 +1,15 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config-open-telemetry.h> + +class FileWatcher { + struct FileInfo { + vespalib::string pathName; + time_t seenModTime; + }; + std::vector<FileInfo> watchedFiles; +public: + bool anyChanged(); + void init(const config::StringVector &pathList); +}; diff --git a/logforwarder/src/apps/vespa-otelcol-start/main.cpp b/logforwarder/src/apps/vespa-otelcol-start/main.cpp new file mode 100644 index 00000000000..2e3e0659707 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/main.cpp @@ -0,0 +1,43 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "wrapper.h" +#include <csignal> +#include <unistd.h> +#include <vespa/vespalib/util/sig_catch.h> + +#include <vespa/defaults.h> +#include <vespa/log/log.h> +LOG_SETUP("vespa-otelcol-start"); + +static void run(const char *configId) { + vespalib::SigCatch catcher; + Wrapper handler(configId); + handler.start(configId); + while (! catcher.receivedStopSignal()) { + handler.check(); + usleep(125000); // Avoid busy looping; + } + handler.stop(); +}; + +int main(int argc, char** argv) { + vespa::Defaults::bootstrap(argv[0]); + int c = -1; + const char *cfid = nullptr; + while ((c = getopt(argc, argv, "c:")) != -1) { + switch (c) { + case 'c': + cfid = optarg; + break; + default: + cfid = nullptr; + break; + } + } + if (cfid == nullptr) { + LOG(error, "Usage: %s -c <config-id>", argv[0]); + return EXIT_FAILURE; + } + run(cfid); + return 0; +} diff --git a/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp b/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp new file mode 100644 index 00000000000..7228a3ea921 --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/wrapper.cpp @@ -0,0 +1,76 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "wrapper.h" +#include "child-handler.h" + +#include <dirent.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include <vespa/defaults.h> +#include <vespa/log/log.h> +LOG_SETUP(".wrapper"); + +namespace { + +vespalib::string fixDir(const vespalib::string &parent, const vespalib::string &subdir) { + auto dirname = parent + "/" + subdir; + DIR *dp = opendir(dirname.c_str()); + if (dp == NULL) { + if (errno != ENOENT || mkdir(dirname.c_str(), 0755) != 0) { + LOG(warning, "Could not create directory '%s'", dirname.c_str()); + perror(dirname.c_str()); + } + } else { + closedir(dp); + } + return dirname; +} + +vespalib::string cfFilePath() { + vespalib::string path = vespa::Defaults::underVespaHome("var/db/vespa"); + path = fixDir(path, "otelcol"); + return path + "/" + "config.yaml"; +} + +void writeConfig(const vespalib::string &config, const vespalib::string &path) { + LOG(info, "got config, writing %s", path.c_str()); + vespalib::string tmpPath = path + ".new"; + FILE *fp = fopen(tmpPath.c_str(), "w"); + if (fp == NULL) { + LOG(warning, "could not open '%s' for write", tmpPath.c_str()); + return; + } + fprintf(fp, "%s\n", config.c_str()); + fclose(fp); + rename(tmpPath.c_str(), path.c_str()); +} + +} // namespace <unnamed> + +Wrapper::Wrapper(const std::string &configId) + : CfHandler(configId), + _childHandler() +{} + +Wrapper::~Wrapper() = default; + +void Wrapper::stop() { + _childHandler.stopChild(); +} + +void Wrapper::check() { + checkConfig(); + if (_childHandler.checkChild()) { + LOG(error, "Fatal: child process died unexpectedly"); + std::_Exit(EXIT_FAILURE); + } +} + +void Wrapper::gotConfig(const OpenTelemetryConfig& config) { + _childHandler.stopChild(); + std::string progPath = vespa::Defaults::underVespaHome("sbin/otelcol-contrib"); + std::string cfPath = cfFilePath(); + writeConfig(config.yaml, cfPath); + _childHandler.startChild(progPath, cfPath); +} diff --git a/logforwarder/src/apps/vespa-otelcol-start/wrapper.h b/logforwarder/src/apps/vespa-otelcol-start/wrapper.h new file mode 100644 index 00000000000..d6ff06a4f0d --- /dev/null +++ b/logforwarder/src/apps/vespa-otelcol-start/wrapper.h @@ -0,0 +1,18 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "cf-handler.h" +#include "child-handler.h" + +#include <string> + +class Wrapper : public CfHandler { +private: + ChildHandler _childHandler; +public: + Wrapper(const std::string &configId); + ~Wrapper(); + void check(); + void stop(); + void gotConfig(const OpenTelemetryConfig& config) override; +}; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java index 4835aad5a27..7316645b5c9 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/AllPassThrottlePolicy.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; /** * This is an implementation of the {@link ThrottlePolicy} that passes all requests (no real throttling). * - * @author dybis + * @author Haakon Dybdahl */ public class AllPassThrottlePolicy implements ThrottlePolicy { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java index 86504d3ad35..d4cec5de323 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java @@ -5,7 +5,6 @@ import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import java.util.Arrays; import java.util.List; /** @@ -76,7 +75,7 @@ public class RPCMessageBus extends NetworkMessageBus { * destination sessions to be routed to. */ public RPCMessageBus(Protocol protocol, String configId) { - this(Arrays.asList(protocol), new RPCNetworkParams().setIdentity(new Identity(configId)), null); + this(List.of(protocol), new RPCNetworkParams().setIdentity(new Identity(configId)), null); } /** diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/test/CustomPolicyFactory.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/test/CustomPolicyFactory.java index 2e568598a94..10a7089fd20 100755..100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/test/CustomPolicyFactory.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/test/CustomPolicyFactory.java @@ -6,7 +6,6 @@ import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.messagebus.test.SimpleProtocol; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -26,7 +25,7 @@ public class CustomPolicyFactory implements SimpleProtocol.PolicyFactory { } public CustomPolicyFactory(boolean selectOnRetry, int consumableError) { - this(selectOnRetry, Arrays.asList(consumableError)); + this(selectOnRetry, List.of(consumableError)); } public CustomPolicyFactory(boolean selectOnRetry, List<Integer> consumableErrors) { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ErrorTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ErrorTestCase.java index 5a3d96991bb..4d51d265cc0 100755..100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/ErrorTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ErrorTestCase.java @@ -9,7 +9,8 @@ import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -33,9 +34,9 @@ public class ErrorTestCase { @Test void requireThatErrorIsPropagated() throws Exception { RoutingTableSpec table = new RoutingTableSpec(SimpleProtocol.NAME); - table.addHop("itr", "test/itr/session", Arrays.asList("test/itr/session")); - table.addHop("dst", "test/dst/session", Arrays.asList("test/dst/session")); - table.addRoute("test", Arrays.asList("itr", "dst")); + table.addHop("itr", "test/itr/session", List.of("test/itr/session")); + table.addHop("dst", "test/dst/session", List.of("test/dst/session")); + table.addRoute("test", List.of("itr", "dst")); Slobrok slobrok = new Slobrok(); TestServer src = new TestServer("test/src", table, slobrok, null); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/TraceTripTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/TraceTripTestCase.java index 7d46add79c9..0aeeba68080 100755..100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/TraceTripTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/TraceTripTestCase.java @@ -12,8 +12,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.net.UnknownHostException; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -31,9 +30,9 @@ public class TraceTripTestCase { @BeforeEach public void setUp() throws ListenFailedException { RoutingTableSpec table = new RoutingTableSpec(SimpleProtocol.NAME) - .addHop("pxy", "test/pxy/session", Arrays.asList("test/pxy/session")) - .addHop("dst", "test/dst/session", Arrays.asList("test/dst/session")) - .addRoute("test", Arrays.asList("pxy", "dst")); + .addHop("pxy", "test/pxy/session", List.of("test/pxy/session")) + .addHop("dst", "test/dst/session", List.of("test/dst/session")) + .addRoute("test", List.of("pxy", "dst")); slobrok = new Slobrok(); src = new TestServer("test/src", table, slobrok, null); @@ -82,7 +81,7 @@ public class TraceTripTestCase { } private static class Proxy implements MessageHandler, ReplyHandler { - private IntermediateSession session; + private final IntermediateSession session; public Proxy(MessageBus bus) { session = bus.createIntermediateSession("session", true, this, this); @@ -102,7 +101,7 @@ public class TraceTripTestCase { } private static class Server implements MessageHandler { - private DestinationSession session; + private final DestinationSession session; public Server(MessageBus bus) { session = bus.createDestinationSession("session", true, this); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/BasicNetworkTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/BasicNetworkTestCase.java index 6612421e437..f62e586d1e2 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/BasicNetworkTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/BasicNetworkTestCase.java @@ -16,7 +16,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -36,7 +35,7 @@ public class BasicNetworkTestCase { RoutingTableSpec table = new RoutingTableSpec(SimpleProtocol.NAME); table.addHop("pxy", "test/pxy/session", List.of("test/pxy/session")); table.addHop("dst", "test/dst/session", List.of("test/dst/session")); - table.addRoute("test", Arrays.asList("pxy", "dst")); + table.addRoute("test", List.of("pxy", "dst")); slobrok = new Slobrok(); src = new TestServer("test/src", table, slobrok, null); pxy = new TestServer("test/pxy", table, slobrok, null); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java index 061416b9eed..c30b4e2aa03 100755..100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.net.UnknownHostException; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -87,7 +86,7 @@ public class SendAdapterTestCase { @Test void requireThatMessagesCanBeSentAcrossAllSupportedVersions() { - List<Version> versions = Arrays.asList( + List<Version> versions = List.of( new Version(6, 149), new Version(9, 999) ); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java index 37d0bbd4c92..ded431cd258 100755..100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -62,8 +61,8 @@ public class RoutingContextTestCase { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory( false, - Arrays.asList("foo", "bar", "baz/cox"), - Arrays.asList("foo", "bar"))); + List.of("foo", "bar", "baz/cox"), + List.of("foo", "bar"))); srcServer.mb.putProtocol(protocol); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("myroute").addHop("myhop")) @@ -83,8 +82,8 @@ public class RoutingContextTestCase { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory( false, - Arrays.asList("foo", "foo/bar", "foo/bar0/baz", "foo/bar1/baz", "foo/bar/baz/cox"), - Arrays.asList("foo/bar0/baz", "foo/bar1/baz"))); + List.of("foo", "foo/bar", "foo/bar0/baz", "foo/bar1/baz", "foo/bar/baz/cox"), + List.of("foo/bar0/baz", "foo/bar1/baz"))); srcServer.mb.putProtocol(protocol); srcServer.setupRouting(new RoutingTableSpec(SimpleProtocol.NAME) .addRoute(new RouteSpec("myroute").addHop("myhop")) diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java index 96250e77d06..992c3e99e87 100755..100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java @@ -6,7 +6,6 @@ import com.yahoo.messagebus.ConfigHandler; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -71,17 +70,17 @@ public class RoutingSpecTestCase { @Test void testApplicationSpec() { assertApplicationSpec(List.of("foo"), - Arrays.asList("foo", + List.of("foo", "*")); assertApplicationSpec(List.of("foo/bar"), - Arrays.asList("foo/bar", + List.of("foo/bar", "foo/*", "*/bar", "*/*")); - assertApplicationSpec(Arrays.asList("foo/0/baz", + assertApplicationSpec(List.of("foo/0/baz", "foo/1/baz", "foo/2/baz"), - Arrays.asList("foo/0/baz", + List.of("foo/0/baz", "foo/1/baz", "foo/2/baz", "foo/0/*", @@ -259,7 +258,7 @@ public class RoutingSpecTestCase { new ApplicationSpec() .addService("mytable", "bar") .addService("mytable", "baz"), - Arrays.asList("Routing table 'mytable' is defined 2 times.", + List.of("Routing table 'mytable' is defined 2 times.", "For hop 'hop2' in routing table 'mytable'; Failed to parse empty string.", "For hop 'hop3' in routing table 'mytable'; Failed to completely parse 'bar/baz cox'.", "For hop 1 in route 'route2' in routing table 'mytable'; Failed to parse empty string.", diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java index 5eac60f4eb2..ac8e1d74191 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.logging.Logger; import java.util.logging.Level; @@ -270,7 +269,7 @@ public class RoutingTestCase { dstSession.acknowledge(msg); assertNotNull(reply = ((Receptor) srcSession.getReplyHandler()).getReply(60)); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList("[APP_TRANSIENT_ERROR @ localhost]: err1", + assertTrace(List.of("[APP_TRANSIENT_ERROR @ localhost]: err1", "-[APP_TRANSIENT_ERROR @ localhost]: err1", "[APP_TRANSIENT_ERROR @ localhost]: err2", "-[APP_TRANSIENT_ERROR @ localhost]: err2"), @@ -298,7 +297,7 @@ public class RoutingTestCase { dstSession.acknowledge(msg); assertNotNull(reply = ((Receptor) srcSession.getReplyHandler()).getReply(60)); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList("Source session accepted a 3 byte message. 1 message(s) now pending.", + assertTrace(List.of("Source session accepted a 3 byte message. 1 message(s) now pending.", "Running routing policy 'Custom'.", "Selecting [" + dstSessName + "].", "Component '" + dstSessName + "' selected by policy 'Custom'.", @@ -371,7 +370,7 @@ public class RoutingTestCase { dstSession.acknowledge(msg); assertNotNull(reply = ((Receptor) srcSession.getReplyHandler()).getReply(60)); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList("Selecting [" + dstSessName + "].", + assertTrace(List.of("Selecting [" + dstSessName + "].", "[APP_TRANSIENT_ERROR @ localhost]", "-[APP_TRANSIENT_ERROR @ localhost]", "Merged [" + dstSessName + "].", @@ -397,7 +396,7 @@ public class RoutingTestCase { dstSession.acknowledge(msg); assertNotNull(reply = ((Receptor) srcSession.getReplyHandler()).getReply(60)); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList("Selecting [" + dstSessName + "].", + assertTrace(List.of("Selecting [" + dstSessName + "].", "[APP_TRANSIENT_ERROR @ localhost]", "-[APP_TRANSIENT_ERROR @ localhost]", "Merged [" + dstSessName + "].", @@ -421,7 +420,7 @@ public class RoutingTestCase { assertNotNull(reply); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.NO_ADDRESS_FOR_SERVICE, reply.getError(0).getCode()); - assertTrace(Arrays.asList("Selecting [" + dstSessName + ", dst/unknown].", + assertTrace(List.of("Selecting [" + dstSessName + ", dst/unknown].", "[NO_ADDRESS_FOR_SERVICE @ localhost]", "Sending reply", "Merged [" + dstSessName + ", dst/unknown]."), @@ -439,7 +438,7 @@ public class RoutingTestCase { assertNotNull(reply); assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.NO_ADDRESS_FOR_SERVICE, reply.getError(0).getCode()); - assertTrace(Arrays.asList("Selecting [dst/unknown].", + assertTrace(List.of("Selecting [dst/unknown].", "[NO_ADDRESS_FOR_SERVICE @ localhost]", "Merged [dst/unknown]."), reply.getTrace()); @@ -479,7 +478,7 @@ public class RoutingTestCase { Reply reply = ((Receptor) srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList("[NO_ADDRESS_FOR_SERVICE @ localhost]", + assertTrace(List.of("[NO_ADDRESS_FOR_SERVICE @ localhost]", "-[NO_ADDRESS_FOR_SERVICE @ localhost]", "Sending message", "-Sending message"), @@ -537,7 +536,7 @@ public class RoutingTestCase { CustomPolicyFactory.parseRoutes(param), 0)); protocol.addPolicyFactory("SetReply", param -> new SetReplyPolicy(false, - Arrays.asList(ErrorCode.APP_TRANSIENT_ERROR, ErrorCode.APP_FATAL_ERROR), + List.of(ErrorCode.APP_TRANSIENT_ERROR, ErrorCode.APP_FATAL_ERROR), param)); srcServer.mb.putProtocol(protocol); assertTrue(srcSession @@ -550,7 +549,7 @@ public class RoutingTestCase { assertEquals(1, reply.getNumErrors()); assertEquals(ErrorCode.APP_FATAL_ERROR, reply.getError(0).getCode()); assertEquals("foo", reply.getError(0).getMessage()); - assertTrace(Arrays.asList("Resolving '[SetReply:foo]'.", + assertTrace(List.of("Resolving '[SetReply:foo]'.", "Resolving '" + dstSessName + "'.", "Resender resending message.", "Resolving '" + dstSessName + "'.", @@ -722,7 +721,7 @@ public class RoutingTestCase { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory(false)); protocol.addPolicyFactory("SetReply", param -> new SetReplyPolicy(false, - Arrays.asList(ErrorCode.APP_TRANSIENT_ERROR, + List.of(ErrorCode.APP_TRANSIENT_ERROR, ErrorCode.APP_TRANSIENT_ERROR, ErrorCode.APP_FATAL_ERROR), param)); @@ -870,7 +869,7 @@ public class RoutingTestCase { Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); assertNotNull(reply); assertFalse(reply.hasErrors()); - assertTrace(Arrays.asList(expectedTrace), reply.getTrace()); + assertTrace(List.of(expectedTrace), reply.getTrace()); } public static void assertTrace(List<String> expected, Trace trace) { diff --git a/messagebus_test/src/tests/error/JavaClient.java b/messagebus_test/src/tests/error/JavaClient.java index 764cf4c1d86..3a420caa6ec 100644 --- a/messagebus_test/src/tests/error/JavaClient.java +++ b/messagebus_test/src/tests/error/JavaClient.java @@ -6,8 +6,8 @@ import com.yahoo.messagebus.routing.*; import com.yahoo.messagebus.network.*; import com.yahoo.messagebus.network.rpc.*; import com.yahoo.messagebus.network.rpc.test.*; -import java.util.Arrays; import java.util.logging.*; +import java.util.List; public class JavaClient { @@ -16,7 +16,7 @@ public class JavaClient { public static void main(String[] args) { try { RPCMessageBus mb = new RPCMessageBus( - Arrays.asList((Protocol)new SimpleProtocol()), + List.of(new SimpleProtocol()), new RPCNetworkParams() .setIdentity(new Identity("server/java")) .setSlobrokConfigId("file:slobrok.cfg"), diff --git a/messagebus_test/src/tests/error/JavaServer.java b/messagebus_test/src/tests/error/JavaServer.java index 7cc366fd7b4..1baa63b0b6e 100644 --- a/messagebus_test/src/tests/error/JavaServer.java +++ b/messagebus_test/src/tests/error/JavaServer.java @@ -5,8 +5,8 @@ import com.yahoo.config.*; import com.yahoo.messagebus.routing.*; import com.yahoo.messagebus.network.*; import com.yahoo.messagebus.network.rpc.*; -import java.util.Arrays; import java.util.logging.*; +import java.util.List; public class JavaServer implements MessageHandler { @@ -29,7 +29,7 @@ public class JavaServer implements MessageHandler { public static void main(String[] args) { try { RPCMessageBus mb = new RPCMessageBus( - Arrays.asList((Protocol)new SimpleProtocol()), + List.of(new SimpleProtocol()), new RPCNetworkParams() .setIdentity(new Identity("server/java")) .setSlobrokConfigId("file:slobrok.cfg"), diff --git a/messagebus_test/src/tests/speed/JavaServer.java b/messagebus_test/src/tests/speed/JavaServer.java index 256f6f00223..58f18237041 100644 --- a/messagebus_test/src/tests/speed/JavaServer.java +++ b/messagebus_test/src/tests/speed/JavaServer.java @@ -5,8 +5,8 @@ import com.yahoo.config.*; import com.yahoo.messagebus.routing.*; import com.yahoo.messagebus.network.*; import com.yahoo.messagebus.network.rpc.*; -import java.util.Arrays; import java.util.logging.*; +import java.util.List; public class JavaServer implements MessageHandler { @@ -36,7 +36,7 @@ public class JavaServer implements MessageHandler { public static void main(String[] args) { try { RPCMessageBus mb = new RPCMessageBus( - Arrays.asList((Protocol)new SimpleProtocol()), + List.of(new SimpleProtocol()), new RPCNetworkParams() .setIdentity(new Identity("server/java")) .setSlobrokConfigId("file:slobrok.cfg"), diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index b610c57cd99..60e85f3f9e4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -13,7 +13,6 @@ import ai.vespa.metricsproxy.service.VespaServices; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -107,7 +106,7 @@ public class MetricsManager { * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary. */ public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime, ConsumerId consumerId) { - if (services.isEmpty()) return Collections.emptyList(); + if (services.isEmpty()) return List.of(); log.log(FINE, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); @@ -164,7 +163,7 @@ public class MetricsManager { * @return Health metrics for all matching services. */ public List<MetricsPacket> getHealthMetrics(List<VespaService> services) { - if (services.isEmpty()) return Collections.emptyList(); + if (services.isEmpty()) return List.of(); vespaServices.updateServices(services); // TODO: Add global dimensions to health metrics? @@ -190,7 +189,7 @@ public class MetricsManager { public void purgeExtraMetrics() { extraDimensions = new HashMap<>(); - externalMetrics.setExtraMetrics(Collections.emptyList()); + externalMetrics.setExtraMetrics(List.of()); } /** diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index 9fe5983d5b9..4dabc7a66d8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -17,7 +17,6 @@ import ai.vespa.metricsproxy.service.VespaService; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -140,7 +139,7 @@ public class VespaMetrics { if ( ! configuredDimensions.isEmpty()) { Map<DimensionId, String> dims = new HashMap<>(dimensions); configuredDimensions.forEach(d -> dims.put(d.key(), d.value())); - dimensions = Collections.unmodifiableMap(dims); + dimensions = Map.copyOf(dims); } return dimensions; } @@ -222,10 +221,10 @@ public class VespaMetrics { } private List<ConfiguredMetric> getMetricDefinitions(ConsumerId consumer) { - if (metricsConsumers == null) return Collections.emptyList(); + if (metricsConsumers == null) return List.of(); List<ConfiguredMetric> definitions = metricsConsumers.getMetricDefinitions(consumer); - return definitions == null ? Collections.emptyList() : definitions; + return definitions == null ? List.of() : definitions; } private static void setMetaInfo(MetricsPacket.Builder builder, Instant timestamp) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index 0277bda078f..58b51020bb9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; -import java.util.stream.Collectors; import static ai.vespa.metricsproxy.http.ValuesFetcher.getConsumerOrDefault; import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java index a6b09ddefd8..fe35822c4af 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.yahoo.concurrent.CopyOnWriteHashMap; import java.util.Map; @@ -13,12 +14,18 @@ public final class DimensionId { private static final Map<String, DimensionId> dictionary = new CopyOnWriteHashMap<>(); public final String id; - private DimensionId(String id) { this.id = id; } + private final String idForPrometheus; + private DimensionId(String id) { + this.id = id; + idForPrometheus = PrometheusUtil.sanitize(id); + } public static DimensionId toDimensionId(String id) { return dictionary.computeIfAbsent(id, key -> new DimensionId(key)); } + public String getIdForPrometheus() { return idForPrometheus; } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java index 9014e818eab..0080ec86272 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.yahoo.concurrent.CopyOnWriteHashMap; import java.util.Map; @@ -14,11 +15,16 @@ public class MetricId { private static final Map<String, MetricId> dictionary = new CopyOnWriteHashMap<>(); public static final MetricId empty = toMetricId(""); public final String id; - private MetricId(String id) { this.id = id; } + private final String idForPrometheus; + private MetricId(String id) { + this.id = id; + idForPrometheus = PrometheusUtil.sanitize(id); + } public static MetricId toMetricId(String id) { - return dictionary.computeIfAbsent(id, key -> new MetricId(key)); + return dictionary.computeIfAbsent(id, MetricId::new); } + public String getIdForPrometheus() { return idForPrometheus; } @Override public boolean equals(Object o) { @@ -29,13 +35,9 @@ public class MetricId { } @Override - public int hashCode() { - return Objects.hash(id); - } + public int hashCode() { return Objects.hash(id); } @Override - public String toString() { - return id; - } + public String toString() { return id; } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 53c6c533518..fb08d78a975 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -14,7 +14,6 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; -import static java.util.Collections.unmodifiableMap; import static java.util.stream.Collectors.joining; /** @@ -42,13 +41,13 @@ public class MetricsPacket { this.statusMessage = statusMessage; this.timestamp = timestamp; this.service = service; - this.metrics = metrics; - this.dimensions = dimensions; + this.metrics = Collections.unmodifiableMap(metrics); // Retain order for tests + this.dimensions = Collections.unmodifiableMap(dimensions); // Retain order for tests this.consumers = Set.copyOf(consumers); } - public Map<MetricId, Number> metrics() { return unmodifiableMap(metrics); } - public Map<DimensionId, String> dimensions() { return unmodifiableMap(dimensions); } + public Map<MetricId, Number> metrics() { return metrics; } + public Map<DimensionId, String> dimensions() { return dimensions; } public Set<ConsumerId> consumers() { return consumers;} @Override @@ -80,7 +79,7 @@ public class MetricsPacket { private long timestamp = 0L; private Map<MetricId, Number> metrics = new LinkedHashMap<>(); private final Map<DimensionId, String> dimensions = new LinkedHashMap<>(); - private Set<ConsumerId> consumers = Collections.emptySet(); + private Set<ConsumerId> consumers = Set.of(); public Builder(ServiceId service) { Objects.requireNonNull(service, "Service cannot be null."); @@ -169,7 +168,7 @@ public class MetricsPacket { if ((extraConsumers != null) && !extraConsumers.isEmpty()) { if (consumers.isEmpty()) { if (extraConsumers.size() == 1) { - consumers = Collections.singleton(extraConsumers.iterator().next()); + consumers = Set.of(extraConsumers.iterator().next()); return this; } consumers = new LinkedHashSet<>(extraConsumers.size()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java index 96ee2fa00e2..20641545a7d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; + import java.util.Objects; /** @@ -9,10 +11,16 @@ import java.util.Objects; public class ServiceId { public final String id; - private ServiceId(String id) { this.id = id; } + private final String idForPrometheus; + private ServiceId(String id) { + this.id = id; + idForPrometheus = PrometheusUtil.sanitize(id); + } public static ServiceId toServiceId(String id) { return new ServiceId(id); } + public String getIdForPrometheus() { return idForPrometheus; } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java index c32e90ecc9f..3d2bf7aaecf 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java @@ -18,8 +18,7 @@ import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.objectMapper; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; + import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; @@ -90,7 +89,7 @@ public class GenericJsonUtil { return toMetricsPackets(jsonModel); } catch (IOException e) { log.log(WARNING, "Could not create metrics packet from string:\n" + jsonString, e); - return emptyList(); + return List.of(); } } @@ -106,7 +105,7 @@ public class GenericJsonUtil { if (node == null) return packets; if (node.metrics == null || node.metrics.isEmpty()) { - return singletonList(new MetricsPacket.Builder(VESPA_NODE_SERVICE_ID) + return List.of(new MetricsPacket.Builder(VESPA_NODE_SERVICE_ID) .statusCode(StatusCode.UP.ordinal()) .timestamp(node.timestamp)); } @@ -124,7 +123,7 @@ public class GenericJsonUtil { private static List<MetricsPacket.Builder> toServicePackets(GenericService service) { List<MetricsPacket.Builder> packets = new ArrayList<>(); if (service.metrics == null || service.metrics.isEmpty()) - return singletonList(newServicePacket(service)); + return List.of(newServicePacket(service)); for (var genericMetrics : service.metrics) { var packet = newServicePacket(service); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java index 3fd95220aca..44f4efb41cc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java @@ -19,9 +19,6 @@ import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; /** * Datamodel for Yamas execute output @@ -80,7 +77,7 @@ public class YamasJsonModel { } List<Metric> getMetricsList() { - if (metrics == null) return emptyList(); + if (metrics == null) return List.of(); return metrics.keySet().stream() .map(name -> new Metric(MetricId.toMetricId(name), metrics.get(name))) @@ -88,14 +85,14 @@ public class YamasJsonModel { } Map<DimensionId, String> getDimensionsById() { - if (dimensions == null) return emptyMap(); + if (dimensions == null) return Map.of(); return dimensions.keySet().stream().collect(toLinkedMap(DimensionId::toDimensionId, name -> dimensions.get(name))); } Set<ConsumerId> getYamasConsumers() { - if (routing == null || routing.get("yamas") == null) return emptySet(); + if (routing == null || routing.get("yamas") == null) return Set.of(); return routing.get("yamas").namespaces.stream() .map(ConsumerId::toConsumerId) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java index 82049da5115..649a4978ed9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonUtil.java @@ -21,7 +21,6 @@ import java.util.stream.Collectors; import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.objectMapper; -import static java.util.Collections.emptyList; import static java.util.logging.Level.WARNING; /** @@ -65,7 +64,7 @@ public class YamasJsonUtil { return packets; } catch (IOException e) { log.log(WARNING, "Could not create metrics packet from string:\n" + jsonString, e); - return emptyList(); + return List.of(); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java index 77db2389271..1f55e2f8679 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusModel.java @@ -1,38 +1,51 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.prometheus; +import ai.vespa.metricsproxy.metric.model.DimensionId; +import ai.vespa.metricsproxy.metric.model.MetricId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.ServiceId; import io.prometheus.client.Collector; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; import io.prometheus.client.exporter.common.TextFormat; -import java.io.IOException; import java.io.StringWriter; +import java.util.ArrayList; import java.util.Enumeration; import java.util.Iterator; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.Map; +import java.util.Set; /** * @author yj-jtakagi * @author gjoranv */ -public class PrometheusModel implements Enumeration<Collector.MetricFamilySamples> { - private static Logger log = Logger.getLogger(PrometheusModel.class.getName()); +public class PrometheusModel implements Enumeration<MetricFamilySamples> { + private final Map<ServiceId, List<MetricsPacket>> packetsByServiceId; + private final Iterator<MetricId> metricIterator; + private final Iterator<MetricFamilySamples> statusMetrics; - private final Iterator<Collector.MetricFamilySamples> metricFamilySamplesIterator; - - PrometheusModel(List<Collector.MetricFamilySamples> metricFamilySamples) { - this.metricFamilySamplesIterator = metricFamilySamples.iterator(); + PrometheusModel(Set<MetricId> metricNames, Map<ServiceId, + List<MetricsPacket>> packetsByServiceId, + List<MetricFamilySamples> statusMetrics) + { + metricIterator = metricNames.iterator(); + this.packetsByServiceId = packetsByServiceId; + this.statusMetrics = statusMetrics.iterator(); } @Override public boolean hasMoreElements() { - return metricFamilySamplesIterator.hasNext(); + return metricIterator.hasNext() || statusMetrics.hasNext(); } @Override - public Collector.MetricFamilySamples nextElement() { - return metricFamilySamplesIterator.next(); + public MetricFamilySamples nextElement() { + return metricIterator.hasNext() + ? createMetricFamily(metricIterator.next()) + : statusMetrics.next(); } public String serialize() { @@ -45,4 +58,31 @@ public class PrometheusModel implements Enumeration<Collector.MetricFamilySample return writer.toString(); } + private MetricFamilySamples createMetricFamily(MetricId metricId) { + List<MetricFamilySamples.Sample> sampleList = new ArrayList<>(); + packetsByServiceId.forEach(((serviceId, packets) -> { + for (var packet : packets) { + Number metric = packet.metrics().get(metricId); + if (metric != null) { + sampleList.add(createSample(serviceId, metricId, metric, packet.timestamp, packet.dimensions())); + } + } + })); + return new MetricFamilySamples(metricId.getIdForPrometheus(), Collector.Type.UNKNOWN, "", sampleList); + } + private static Sample createSample(ServiceId serviceId, MetricId metricId, Number metric, + Long timeStamp, Map<DimensionId, String> dimensions) + { + List<String> labels = new ArrayList<>(dimensions.size()); + List<String> labelValues = new ArrayList<>(dimensions.size()); + for (var entry : dimensions.entrySet()) { + var labelName = entry.getKey().getIdForPrometheus(); + labels.add(labelName); + labelValues.add(entry.getValue()); + } + labels.add("vespa_service"); + labelValues.add(serviceId.getIdForPrometheus()); + return new Sample(metricId.getIdForPrometheus(), labels, labelValues, metric.doubleValue(), timeStamp); + } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java index 973e1adb96d..47b97fa0902 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java @@ -1,74 +1,53 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.prometheus; +import ai.vespa.metricsproxy.metric.model.MetricId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.ServiceId; import io.prometheus.client.Collector; import io.prometheus.client.Collector.MetricFamilySamples; -import io.prometheus.client.Collector.MetricFamilySamples.Sample; import java.util.ArrayList; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; - /** * @author yj-jtakagi * @author gjoranv */ public class PrometheusUtil { + public static String sanitize(String name) { + String sanitized = Collector.sanitizeMetricName(name); + return name.equals(sanitized) ? name : sanitized; + } + public static PrometheusModel toPrometheusModel(List<MetricsPacket> metricsPackets) { + Set<MetricId> metricNames = new HashSet<>(); + for (MetricsPacket metricsPacket : metricsPackets) { + metricNames.addAll(metricsPacket.metrics().keySet()); + } + Map<ServiceId, List<MetricsPacket>> packetsByService = metricsPackets.stream() .collect(Collectors.groupingBy(packet -> packet.service)); - List<MetricFamilySamples> metricFamilySamples = new ArrayList<>(packetsByService.size()); - - Map<String, List<Sample>> samples = new HashMap<>(); + List<MetricFamilySamples> statusMetrics = new ArrayList<>(packetsByService.size()); packetsByService.forEach(((serviceId, packets) -> { - - var serviceName = Collector.sanitizeMetricName(serviceId.id); - for (var packet : packets) { - var dimensions = packet.dimensions(); - List<String> labels = new ArrayList<>(dimensions.size()); - List<String> labelValues = new ArrayList<>(dimensions.size()); - for (var entry : dimensions.entrySet()) { - var labelName = Collector.sanitizeMetricName(entry.getKey().id); - labels.add(labelName); - labelValues.add(entry.getValue()); - } - labels.add("vespa_service"); - labelValues.add(serviceName); - - for (var metric : packet.metrics().entrySet()) { - var metricName = Collector.sanitizeMetricName(metric.getKey().id); - List<Sample> sampleList; - if (samples.containsKey(metricName)) { - sampleList = samples.get(metricName); - } else { - sampleList = new ArrayList<>(); - samples.put(metricName, sampleList); - metricFamilySamples.add(new MetricFamilySamples(metricName, Collector.Type.UNKNOWN, "", sampleList)); - } - sampleList.add(new Sample(metricName, labels, labelValues, metric.getValue().doubleValue(), packet.timestamp * 1000)); - } - } + var serviceName = serviceId.getIdForPrometheus(); if (!packets.isEmpty()) { var firstPacket = packets.get(0); var statusMetricName = serviceName + "_status"; // MetricsPacket status 0 means OK, but it's the opposite in Prometheus. var statusMetricValue = (firstPacket.statusCode == 0) ? 1 : 0; - var sampleList = singletonList(new Sample(statusMetricName, emptyList(), emptyList(), + var sampleList = List.of(new Collector.MetricFamilySamples.Sample(statusMetricName, List.of(), List.of(), statusMetricValue, firstPacket.timestamp * 1000)); - metricFamilySamples.add(new MetricFamilySamples(statusMetricName, Collector.Type.UNKNOWN, "status of service", sampleList)); + statusMetrics.add(new Collector.MetricFamilySamples(statusMetricName, Collector.Type.UNKNOWN, "status of service", sampleList)); } })); - - return new PrometheusModel(metricFamilySamples); + return new PrometheusModel(metricNames, packetsByService, statusMetrics); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index 6c3b759e97b..0e33d7dbf2f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -28,6 +28,7 @@ import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; * @author Jo Kristian Bergum */ public class MetricsParser { + private static final Double ZERO_DOUBLE = 0d; public interface Collector { void accept(Metric metric); } @@ -186,7 +187,8 @@ public class MetricsParser { if (token == JsonToken.VALUE_NUMBER_INT) { metrics.add(Map.entry(metricName, parser.getLongValue())); } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { - metrics.add(Map.entry(metricName, parser.getValueAsDouble())); + double value = parser.getValueAsDouble(); + metrics.add(Map.entry(metricName, value == ZERO_DOUBLE ? ZERO_DOUBLE : value)); } else { throw new IllegalArgumentException("Value for aggregator '" + fieldName + "' is not a number"); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java index a83a28ff9c6..573292a6592 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java @@ -7,7 +7,6 @@ import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.ServiceId; -import java.util.Collections; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -20,7 +19,7 @@ import java.util.concurrent.atomic.AtomicReference; */ public class VespaService implements Comparable<VespaService> { - private static final Map<DimensionId, String> EMPTY_DIMENSIONS = Collections.emptyMap(); + private static final Map<DimensionId, String> EMPTY_DIMENSIONS = Map.of(); private static final String DEFAULT_MONITORING_PREFIX = "vespa"; public static final String SEPARATOR = "."; diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index 05d51967166..dc461fbdbfa 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -45,8 +45,8 @@ public enum ControllerMetrics implements VespaMetrics { AUTH0_EXCEPTIONS("auth0.exceptions", Unit.FAILURE, "Controller: Auth0 exceptions"), CERTIFICATE_POOL_AVAILABLE("certificate_pool_available", Unit.FRACTION, "Available certificates in the pool, fraction of configured size"), BILLING_EXCEPTIONS("billing.exceptions", Unit.FAILURE, "Controller: Billing related exceptions"), - BILLING_WEBHOOK_FILTER_FAILURES("billing.webhook.failures", Unit.FAILURE, "Controller: webhook filter failures"), - BILLING_WEBHOOK_FILTER_REQUESTS("billing.webhook.requests", Unit.REQUEST, "Controller: webhook filter requests"), + BILLING_WEBHOOK_FAILURES("billing.webhook.failures", Unit.FAILURE, "Controller: webhook failures"), + BILLING_WEBHOOK_REQUESTS("billing.webhook.requests", Unit.REQUEST, "Controller: webhook requests"), // Metrics per API, metrics names generated in ControllerMaintainer/MetricsReporter OPERATION_APPLICATION("operation.application", Unit.REQUEST, "Controller: Requests for /application API"), diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 9479c814e89..8c500473678 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -185,6 +185,7 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.ZMS_QUOTA_USAGE.max()); addMetric(metrics, ControllerMetrics.COREDUMP_PROCESSED.count()); addMetric(metrics, ControllerMetrics.AUTH0_EXCEPTIONS.count()); + addMetric(metrics, ControllerMetrics.BILLING_WEBHOOK_FAILURES.count()); addMetric(metrics, ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.max()); addMetric(metrics, ControllerMetrics.BILLING_EXCEPTIONS.count()); diff --git a/metrics/src/test/java/ai/vespa/metrics/MetricSetTest.java b/metrics/src/test/java/ai/vespa/metrics/MetricSetTest.java index db6f5457b5b..962d3c1f1d6 100644 --- a/metrics/src/test/java/ai/vespa/metrics/MetricSetTest.java +++ b/metrics/src/test/java/ai/vespa/metrics/MetricSetTest.java @@ -10,7 +10,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; -import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -23,7 +22,7 @@ public class MetricSetTest { void metrics_from_children_are_added() { MetricSet child1 = new MetricSet("child1", List.of(new Metric("child1_metric"))); MetricSet child2 = new MetricSet("child2", List.of(new Metric("child2_metric"))); - MetricSet parent = new MetricSet("parent", emptyList(), List.of(child1, child2)); + MetricSet parent = new MetricSet("parent", List.of(), List.of(child1, child2)); Map<String, Metric> parentMetrics = parent.getMetrics(); assertEquals(2, parentMetrics.size()); @@ -34,7 +33,7 @@ public class MetricSetTest { @Test void adding_the_same_child_set_twice_has_no_effect() { MetricSet child = new MetricSet("child", List.of(new Metric("child_metric"))); - MetricSet parent = new MetricSet("parent", emptyList(), List.of(child, child)); + MetricSet parent = new MetricSet("parent", List.of(), List.of(child, child)); Map<String, Metric> parentMetrics = parent.getMetrics(); assertEquals(1, parentMetrics.size()); diff --git a/metrics/src/vespa/metrics/metrictimer.cpp b/metrics/src/vespa/metrics/metrictimer.cpp index 84d4844104d..a3b0f215d58 100644 --- a/metrics/src/vespa/metrics/metrictimer.cpp +++ b/metrics/src/vespa/metrics/metrictimer.cpp @@ -3,13 +3,18 @@ namespace metrics { -MetricTimer::MetricTimer() +MetricTimer::MetricTimer() noexcept + : _startTime(std::chrono::steady_clock::now()) { // Amusingly enough, steady_clock was not actually steady by default on // GCC < 4.8.1, so add a bit of compile-time paranoia just to make sure. static_assert(std::chrono::steady_clock::is_steady, "Old/broken STL implementation; steady_clock not steady"); - _startTime = std::chrono::steady_clock::now(); +} + +MetricTimer::MetricTimer(std::chrono::steady_clock::time_point start_time) noexcept + : _startTime(start_time) +{ } } // metrics diff --git a/metrics/src/vespa/metrics/metrictimer.h b/metrics/src/vespa/metrics/metrictimer.h index 8a338432362..133cd819489 100644 --- a/metrics/src/vespa/metrics/metrictimer.h +++ b/metrics/src/vespa/metrics/metrictimer.h @@ -15,7 +15,19 @@ namespace metrics { class MetricTimer { public: - MetricTimer(); + // Start time point set by system steady clock + MetricTimer() noexcept; + // Start time point explicitly given + explicit MetricTimer(std::chrono::steady_clock::time_point start_time) noexcept; + + template<typename AvgVal, typename TotVal, bool SumOnAdd> + AvgVal stop(std::chrono::steady_clock::time_point now, ValueMetric<AvgVal, TotVal, SumOnAdd>& metric) const { + const auto delta = now - _startTime; + using ToDuration = std::chrono::duration<AvgVal, std::milli>; + const auto deltaMs(std::chrono::duration_cast<ToDuration>(delta).count()); + metric.addValue(deltaMs); + return deltaMs; + } /** * Adds ms passed since this timer was constructed to given value metric. @@ -26,11 +38,11 @@ public: */ template<typename AvgVal, typename TotVal, bool SumOnAdd> AvgVal stop(ValueMetric<AvgVal, TotVal, SumOnAdd>& metric) const { - const auto delta = std::chrono::steady_clock::now() - _startTime; - using ToDuration = std::chrono::duration<AvgVal, std::milli>; - const auto deltaMs(std::chrono::duration_cast<ToDuration>(delta).count()); - metric.addValue(deltaMs); - return deltaMs; + return stop(std::chrono::steady_clock::now(), metric); + } + + [[nodiscard]] std::chrono::steady_clock::time_point start_time() const noexcept { + return _startTime; } private: diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 0e2a27952ef..e396020ad7a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -104,7 +103,7 @@ public class RankProfilesConfigImporter { if (externalReference.isPresent()) { RankingExpression expression = largeExpressions.get(property.value()); ExpressionFunction function = new ExpressionFunction(externalReference.get().functionName(), - Collections.emptyList(), + List.of(), expression); if (externalReference.get().isFree()) // make available in model under configured name @@ -115,7 +114,7 @@ public class RankProfilesConfigImporter { else if (reference.isPresent()) { RankingExpression expression = new RankingExpression(reference.get().functionName(), property.value()); ExpressionFunction function = new ExpressionFunction(reference.get().functionName(), - Collections.emptyList(), + List.of(), expression); if (reference.get().isFree()) // make available in model under configured name diff --git a/model-integration/abi-spec.json b/model-integration/abi-spec.json index d3c472778e6..e7130d9c777 100644 --- a/model-integration/abi-spec.json +++ b/model-integration/abi-spec.json @@ -1,4 +1,186 @@ { + "ai.vespa.llm.clients.ConfigurableLanguageModel" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "ai.vespa.llm.LanguageModel" + ], + "attributes" : [ + "public", + "abstract" + ], + "methods" : [ + "public void <init>()", + "public void <init>(ai.vespa.llm.clients.LlmClientConfig, com.yahoo.container.jdisc.secretstore.SecretStore)", + "protected java.lang.String getApiKey(ai.vespa.llm.InferenceParameters)", + "protected void setApiKey(ai.vespa.llm.InferenceParameters)", + "protected java.lang.String getEndpoint()", + "protected void setEndpoint(ai.vespa.llm.InferenceParameters)" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.LlmClientConfig$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>()", + "public void <init>(ai.vespa.llm.clients.LlmClientConfig)", + "public ai.vespa.llm.clients.LlmClientConfig$Builder apiKeySecretName(java.lang.String)", + "public ai.vespa.llm.clients.LlmClientConfig$Builder endpoint(java.lang.String)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public ai.vespa.llm.clients.LlmClientConfig build()" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.LlmClientConfig$Producer" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void getConfig(ai.vespa.llm.clients.LlmClientConfig$Builder)" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.LlmClientConfig" : { + "superClass" : "com.yahoo.config.ConfigInstance", + "interfaces" : [ ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public void <init>(ai.vespa.llm.clients.LlmClientConfig$Builder)", + "public java.lang.String apiKeySecretName()", + "public java.lang.String endpoint()" + ], + "fields" : [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "ai.vespa.llm.clients.LlmLocalClientConfig$Builder" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public void <init>()", + "public void <init>(ai.vespa.llm.clients.LlmLocalClientConfig)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder model(com.yahoo.config.ModelReference)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder parallelRequests(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxQueueSize(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder useGpu(boolean)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder gpuLayers(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder threads(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder contextSize(int)", + "public ai.vespa.llm.clients.LlmLocalClientConfig$Builder maxTokens(int)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public ai.vespa.llm.clients.LlmLocalClientConfig build()" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.LlmLocalClientConfig$Producer" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes" : [ + "public", + "interface", + "abstract" + ], + "methods" : [ + "public abstract void getConfig(ai.vespa.llm.clients.LlmLocalClientConfig$Builder)" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.LlmLocalClientConfig" : { + "superClass" : "com.yahoo.config.ConfigInstance", + "interfaces" : [ ], + "attributes" : [ + "public", + "final" + ], + "methods" : [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public void <init>(ai.vespa.llm.clients.LlmLocalClientConfig$Builder)", + "public java.nio.file.Path model()", + "public int parallelRequests()", + "public int maxQueueSize()", + "public boolean useGpu()", + "public int gpuLayers()", + "public int threads()", + "public int contextSize()", + "public int maxTokens()" + ], + "fields" : [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "ai.vespa.llm.clients.LocalLLM" : { + "superClass" : "com.yahoo.component.AbstractComponent", + "interfaces" : [ + "ai.vespa.llm.LanguageModel" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(ai.vespa.llm.clients.LlmLocalClientConfig)", + "public void deconstruct()", + "public java.util.List complete(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters)", + "public java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters, java.util.function.Consumer)" + ], + "fields" : [ ] + }, + "ai.vespa.llm.clients.OpenAI" : { + "superClass" : "ai.vespa.llm.clients.ConfigurableLanguageModel", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(ai.vespa.llm.clients.LlmClientConfig, com.yahoo.container.jdisc.secretstore.SecretStore)", + "public java.util.List complete(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters)", + "public java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, ai.vespa.llm.InferenceParameters, java.util.function.Consumer)" + ], + "fields" : [ ] + }, "ai.vespa.llm.generation.Generator" : { "superClass" : "com.yahoo.component.AbstractComponent", "interfaces" : [ ], diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 0bab30e1453..d92fa319251 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -40,6 +40,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>searchcore</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -76,6 +82,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-llama</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>component</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java index f43f3834a65..2fd8e312a7e 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/ColBertEmbedder.java @@ -149,7 +149,7 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { } protected TransformerInput buildTransformerInput(List<Long> tokens, int maxTokens, boolean isQuery) { - if(!isQuery) { + if (!isQuery) { tokens = tokens.stream().filter(token -> !skipTokens.contains(token)).toList(); } List<Long> inputIds = new ArrayList<>(maxTokens); @@ -172,7 +172,7 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { attentionMask.add((long) 1); for (int i = 0; i < padding; i++) - attentionMask.add((long) 0);//Do not attend to mask paddings + attentionMask.add((long) 0); // Do not attend to mask paddings return new TransformerInput(inputIds, attentionMask); } @@ -181,56 +181,44 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { if (tensorType.valueType() == TensorType.Value.INT8) throw new IllegalArgumentException("ColBert query embed does not accept int8 tensor value type"); + EmbeddingResult result = lookupOrEvaluate(context, text, true); + return toFloatTensor((IndexedTensor)result.outputs.get(outputName), tensorType, result.inputIdSize); + } - var start = System.nanoTime(); - var encoding = tokenizer.encode(text, context.getLanguage()); - runtime.sampleSequenceLength(encoding.ids().size(), context); - - TransformerInput input = buildTransformerInput(encoding.ids(), maxQueryTokens, true); - - Tensor inputIdsTensor = createTensorRepresentation(input.inputIds, "d1"); - Tensor attentionMaskTensor = createTensorRepresentation(input.attentionMask, "d1"); - - var inputs = Map.of(inputIdsName, inputIdsTensor.expand("d0"), - attentionMaskName, attentionMaskTensor.expand("d0")); - Map<String, Tensor> outputs = evaluator.evaluate(inputs); - Tensor tokenEmbeddings = outputs.get(outputName); - IndexedTensor result = (IndexedTensor) tokenEmbeddings; + protected Tensor embedDocument(String text, Context context, TensorType tensorType) { + EmbeddingResult result = lookupOrEvaluate(context, text, false); + var modelOutput = (IndexedTensor)result.outputs.get(outputName); + if (tensorType.valueType() == TensorType.Value.INT8) + return toBitTensor(modelOutput, tensorType, result.inputIdSize); + else + return toFloatTensor(modelOutput, tensorType, result.inputIdSize); + } - int dims = tensorType.indexedSubtype().dimensions().get(0).size().get().intValue(); - if (dims != result.shape()[2]) { - throw new IllegalArgumentException("Token vector dimensionality does not" + - " match indexed dimensionality of " + dims); - } - Tensor resultTensor = toFloatTensor(result, tensorType, input.inputIds.size()); - runtime.sampleEmbeddingLatency((System.nanoTime() - start) / 1_000_000d, context); - return resultTensor; + /** + * Evaluate the embedding model if the result is not present in the context cache. + * + * @param context the context accompanying the request + * @param text the text that is embedded + * @return the model output + */ + protected EmbeddingResult lookupOrEvaluate(Context context, String text, boolean isQuery) { + var key = new EmbedderCacheKey(context.getEmbedderId(), text); + return context.computeCachedValueIfAbsent(key, () -> evaluate(context, text, isQuery)); } - protected Tensor embedDocument(String text, Context context, TensorType tensorType) { + private EmbeddingResult evaluate(Context context, String text, boolean isQuery) { var start = System.nanoTime(); var encoding = tokenizer.encode(text, context.getLanguage()); runtime.sampleSequenceLength(encoding.ids().size(), context); - - TransformerInput input = buildTransformerInput(encoding.ids(), maxDocumentTokens, false); + TransformerInput input = buildTransformerInput(encoding.ids(), isQuery ? maxQueryTokens : maxDocumentTokens, isQuery); Tensor inputIdsTensor = createTensorRepresentation(input.inputIds, "d1"); Tensor attentionMaskTensor = createTensorRepresentation(input.attentionMask, "d1"); - - var inputs = Map.of(inputIdsName, inputIdsTensor.expand("d0"), + var inputs = Map.of(inputIdsName, + inputIdsTensor.expand("d0"), attentionMaskName, attentionMaskTensor.expand("d0")); - Map<String, Tensor> outputs = evaluator.evaluate(inputs); - Tensor tokenEmbeddings = outputs.get(outputName); - IndexedTensor result = (IndexedTensor) tokenEmbeddings; - Tensor contextualEmbeddings; - int maxTokens = input.inputIds.size(); // Retain all token vectors, including PAD tokens. - if (tensorType.valueType() == TensorType.Value.INT8) { - contextualEmbeddings = toBitTensor(result, tensorType, maxTokens); - } else { - contextualEmbeddings = toFloatTensor(result, tensorType, maxTokens); - } runtime.sampleEmbeddingLatency((System.nanoTime() - start) / 1_000_000d, context); - return contextualEmbeddings; + return new EmbeddingResult(input.inputIds.size(), outputs); } public static Tensor toFloatTensor(IndexedTensor result, TensorType type, int nTokens) { @@ -241,13 +229,13 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { throw new IllegalArgumentException("Target indexed sub-type must have one dimension"); int wantedDimensionality = type.indexedSubtype().dimensions().get(0).size().get().intValue(); int resultDimensionality = (int)result.shape()[2]; - if (resultDimensionality != wantedDimensionality) { + if (wantedDimensionality > resultDimensionality) { throw new IllegalArgumentException("Not possible to map token vector embedding with " + resultDimensionality + " dimensions into tensor with " + wantedDimensionality); } Tensor.Builder builder = Tensor.Builder.of(type); for (int token = 0; token < nTokens; token++) { - for (int d = 0; d < resultDimensionality; d++) { + for (int d = 0; d < wantedDimensionality; d++) { var value = result.get(0,token,d); // batch, sequence token, dimension builder.cell(TensorAddress.of(token,d),value); } @@ -265,8 +253,10 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { if (size != 1) throw new IllegalArgumentException("Target indexed sub-type must have one dimension"); int wantedDimensionality = type.indexedSubtype().dimensions().get(0).size().get().intValue(); + //Allow using the first n float dimensions to pack into int8 + int floatDimensionality = 8 * wantedDimensionality; int resultDimensionality = (int)result.shape()[2]; - if (resultDimensionality != 8 * wantedDimensionality) { + if (floatDimensionality > resultDimensionality) { throw new IllegalArgumentException("Not possible to pack " + resultDimensionality + " + dimensions into " + wantedDimensionality + " dimensions"); } @@ -274,7 +264,7 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { for (int token = 0; token < nTokens; token++) { BitSet bitSet = new BitSet(8); int key = 0; - for (int d = 0; d < result.shape()[2]; d++) { + for (int d = 0; d < floatDimensionality; d++) { var value = result.get(0, token, d); // batch, sequence token, dimension int bitIndex = 7 - (d % 8); if (value > 0.0) { @@ -319,4 +309,8 @@ public class ColBertEmbedder extends AbstractComponent implements Embedder { return builder.build(); } + record EmbedderCacheKey(String embedderId, Object embeddedValue) { } + + record EmbeddingResult(int inputIdSize, Map<String, Tensor> outputs) { } + } diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java index 169648967d7..20d8b6362d3 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java @@ -104,59 +104,23 @@ public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { tokenizer.close(); } + @SuppressWarnings("unchecked") @Override - public Tensor embed(String s, Context context, TensorType tensorType) { - var start = System.nanoTime(); - var encoding = tokenizer.encode(s, context.getLanguage()); - runtime.sampleSequenceLength(encoding.ids().size(), context); - Tensor inputSequence = createTensorRepresentation(encoding.ids(), "d1"); - Tensor attentionMask = createTensorRepresentation(encoding.attentionMask(), "d1"); - Tensor tokenTypeIds = tokenTypeIdsName.isEmpty() ? null : createTensorRepresentation(encoding.typeIds(), "d1"); - - - Map<String, Tensor> inputs; - if (tokenTypeIdsName.isEmpty() || tokenTypeIds.isEmpty()) { - inputs = Map.of(inputIdsName, inputSequence.expand("d0"), - attentionMaskName, attentionMask.expand("d0")); - } else { - inputs = Map.of(inputIdsName, inputSequence.expand("d0"), - attentionMaskName, attentionMask.expand("d0"), - tokenTypeIdsName, tokenTypeIds.expand("d0")); + public Tensor embed(String text, Context context, TensorType tensorType) { + if (tensorType.dimensions().size() != 1) { + throw new IllegalArgumentException("Error in embedding to type '" + tensorType + "': should only have one dimension."); } - - Map<String, Tensor> outputs = evaluator.evaluate(inputs); - IndexedTensor tokenEmbeddings = (IndexedTensor) outputs.get(outputName); - long[] resultShape = tokenEmbeddings.shape(); - //shape batch, sequence, embedding dimensionality - if (resultShape.length != 3) { - throw new IllegalArgumentException("" + - "Expected 3 output dimensions for output name '" + - outputName + "': [batch, sequence, embedding], got " + resultShape.length); + if (!tensorType.dimensions().get(0).isIndexed()) { + throw new IllegalArgumentException("Error in embedding to type '" + tensorType + "': dimension should be indexed."); } - Tensor result; + var embeddingResult = lookupOrEvaluate(context, text); + IndexedTensor tokenEmbeddings = embeddingResult.output; if (tensorType.valueType() == TensorType.Value.INT8) { - long outputDimensions = resultShape[2]; - long targetDim = tensorType.dimensions().get(0).size().get(); - - if(targetDim * 8 > outputDimensions) { - throw new IllegalArgumentException("Cannot pack " + outputDimensions + " into " + targetDim + " int8s"); - } - //Dimensionality flexibility 🪆 - packing only the first 8*targetDim values from the model output - long firstDimensions = 8 * targetDim; - String name = tensorType.indexedSubtype().dimensions().get(0).name(); - //perform pooling and normalizing using floating point embeddings before binarizing - //using the firstDimensions as the target dimensionality - TensorType poolingType = new TensorType.Builder(TensorType.Value.FLOAT).indexed(name, firstDimensions).build(); - result = poolingStrategy.toSentenceEmbedding(poolingType, tokenEmbeddings, attentionMask); - result = normalize? normalize(result, poolingType) : result; - result = binarize((IndexedTensor) result, tensorType); - - } else { // regular floating points embeddings - result = poolingStrategy.toSentenceEmbedding(tensorType, tokenEmbeddings, attentionMask); - result = normalize ? normalize(result, tensorType) : result; + return binaryQuantization(embeddingResult, tensorType); + } else { + Tensor result = poolingStrategy.toSentenceEmbedding(tensorType, tokenEmbeddings, embeddingResult.attentionMask); + return normalize ? normalize(result, tensorType) : result; } - runtime.sampleEmbeddingLatency((System.nanoTime() - start)/1_000_000d, context); - return result; } Tensor normalize(Tensor embedding, TensorType tensorType) { @@ -178,6 +142,61 @@ public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { return builder.build(); } + private HuggingFaceEmbedder.HFEmbeddingResult lookupOrEvaluate(Context context, String text) { + var key = new HFEmbedderCacheKey(context.getEmbedderId(), text); + return context.computeCachedValueIfAbsent(key, () -> evaluate(context, text)); + } + + private HuggingFaceEmbedder.HFEmbeddingResult evaluate(Context context, String text) { + var start = System.nanoTime(); + var encoding = tokenizer.encode(text, context.getLanguage()); + runtime.sampleSequenceLength(encoding.ids().size(), context); + Tensor inputSequence = createTensorRepresentation(encoding.ids(), "d1"); + Tensor attentionMask = createTensorRepresentation(encoding.attentionMask(), "d1"); + Tensor tokenTypeIds = tokenTypeIdsName.isEmpty() ? null : createTensorRepresentation(encoding.typeIds(), "d1"); + + Map<String, Tensor> inputs; + if (tokenTypeIdsName.isEmpty() || tokenTypeIds.isEmpty()) { + inputs = Map.of(inputIdsName, inputSequence.expand("d0"), + attentionMaskName, attentionMask.expand("d0")); + } else { + inputs = Map.of(inputIdsName, inputSequence.expand("d0"), + attentionMaskName, attentionMask.expand("d0"), + tokenTypeIdsName, tokenTypeIds.expand("d0")); + } + IndexedTensor tokenEmbeddings = (IndexedTensor) evaluator.evaluate(inputs).get(outputName); + long[] resultShape = tokenEmbeddings.shape(); + //shape batch, sequence, embedding dimensionality + if (resultShape.length != 3) { + throw new IllegalArgumentException("" + + "Expected 3 output dimensions for output name '" + + outputName + "': [batch, sequence, embedding], got " + resultShape.length); + } + runtime.sampleEmbeddingLatency((System.nanoTime() - start)/1_000_000d, context); + return new HFEmbeddingResult(tokenEmbeddings, attentionMask, context.getEmbedderId()); + } + + private Tensor binaryQuantization(HuggingFaceEmbedder.HFEmbeddingResult embeddingResult, TensorType tensorType) { + long outputDimensions = embeddingResult.output().shape()[2]; + long targetDim = tensorType.dimensions().get(0).size().get(); + //🪆 flexibility - packing only the first 8*targetDim float values from the model output + long floatDimensions = 8 * targetDim; + if(floatDimensions > outputDimensions) { + throw new IllegalArgumentException("Cannot pack " + outputDimensions + " into " + targetDim + " int8s"); + } + //perform pooling and normalizing using float version before binary quantization + TensorType poolingType = new TensorType.Builder(TensorType.Value.FLOAT). + indexed(tensorType.indexedSubtype().dimensions().get(0).name(), + floatDimensions).build(); + Tensor result = poolingStrategy.toSentenceEmbedding(poolingType, embeddingResult.output(), embeddingResult.attentionMask()); + result = normalize? normalize(result, poolingType) : result; + result = binarize((IndexedTensor) result, tensorType); + return result; + } + + /** + * Binary quantization of the embedding into a tensor of type int8 with the specified dimensions. + */ static public Tensor binarize(IndexedTensor embedding, TensorType tensorType) { Tensor.Builder builder = Tensor.Builder.of(tensorType); BitSet bitSet = new BitSet(8); @@ -211,6 +230,7 @@ public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { return builder.build(); } - + protected record HFEmbeddingResult(IndexedTensor output, Tensor attentionMask, String embedderId) {} + protected record HFEmbedderCacheKey(String embedderId, Object embeddedValue) { } } diff --git a/container-search/src/main/java/ai/vespa/llm/clients/ConfigurableLanguageModel.java b/model-integration/src/main/java/ai/vespa/llm/clients/ConfigurableLanguageModel.java index 761fdf0af93..761fdf0af93 100644 --- a/container-search/src/main/java/ai/vespa/llm/clients/ConfigurableLanguageModel.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/ConfigurableLanguageModel.java diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java new file mode 100644 index 00000000000..aa7c071b93a --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java @@ -0,0 +1,125 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.llm.clients; + +import ai.vespa.llm.InferenceParameters; +import ai.vespa.llm.LanguageModel; +import ai.vespa.llm.completion.Completion; +import ai.vespa.llm.completion.Prompt; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import de.kherud.llama.LlamaModel; +import de.kherud.llama.ModelParameters; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.logging.Logger; + +/** + * A language model running locally on the container node. + * + * @author lesters + */ +public class LocalLLM extends AbstractComponent implements LanguageModel { + + private final static Logger logger = Logger.getLogger(LocalLLM.class.getName()); + private final LlamaModel model; + private final ThreadPoolExecutor executor; + private final int contextSize; + private final int maxTokens; + + @Inject + public LocalLLM(LlmLocalClientConfig config) { + executor = createExecutor(config); + + // Maximum number of tokens to generate - need this since some models can just generate infinitely + maxTokens = config.maxTokens(); + + // Only used if GPU is not used + var defaultThreadCount = Math.max(Runtime.getRuntime().availableProcessors() - 2, 1); + + var modelFile = config.model().toFile().getAbsolutePath(); + var modelParams = new ModelParameters() + .setModelFilePath(modelFile) + .setContinuousBatching(true) + .setNParallel(config.parallelRequests()) + .setNThreads(config.threads() <= 0 ? defaultThreadCount : config.threads()) + .setNCtx(config.contextSize()) + .setNGpuLayers(config.useGpu() ? config.gpuLayers() : 0); + + long startLoad = System.nanoTime(); + model = new LlamaModel(modelParams); + long loadTime = System.nanoTime() - startLoad; + logger.info(String.format("Loaded model %s in %.2f sec", modelFile, (loadTime*1.0/1000000000))); + + // Todo: handle prompt context size - such as give a warning when prompt exceeds context size + contextSize = config.contextSize(); + } + + private ThreadPoolExecutor createExecutor(LlmLocalClientConfig config) { + return new ThreadPoolExecutor(config.parallelRequests(), config.parallelRequests(), + 0L, TimeUnit.MILLISECONDS, + config.maxQueueSize() > 0 ? new ArrayBlockingQueue<>(config.maxQueueSize()) : new SynchronousQueue<>(), + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public void deconstruct() { + logger.info("Closing LLM model..."); + model.close(); + executor.shutdownNow(); + } + + @Override + public List<Completion> complete(Prompt prompt, InferenceParameters options) { + StringBuilder result = new StringBuilder(); + var future = completeAsync(prompt, options, completion -> { + result.append(completion.text()); + }).exceptionally(exception -> Completion.FinishReason.error); + var reason = future.join(); + + List<Completion> completions = new ArrayList<>(); + completions.add(new Completion(result.toString(), reason)); + return completions; + } + + @Override + public CompletableFuture<Completion.FinishReason> completeAsync(Prompt prompt, InferenceParameters options, Consumer<Completion> consumer) { + var inferParams = new de.kherud.llama.InferenceParameters(prompt.asString().stripLeading()); + + // We always set this to some value to avoid infinite token generation + inferParams.setNPredict(maxTokens); + + options.ifPresent("temperature", (v) -> inferParams.setTemperature(Float.parseFloat(v))); + options.ifPresent("topk", (v) -> inferParams.setTopK(Integer.parseInt(v))); + options.ifPresent("topp", (v) -> inferParams.setTopP(Integer.parseInt(v))); + options.ifPresent("npredict", (v) -> inferParams.setNPredict(Integer.parseInt(v))); + options.ifPresent("repeatpenalty", (v) -> inferParams.setRepeatPenalty(Float.parseFloat(v))); + // Todo: more options? + + var completionFuture = new CompletableFuture<Completion.FinishReason>(); + try { + executor.submit(() -> { + for (LlamaModel.Output output : model.generate(inferParams)) { + consumer.accept(Completion.from(output.text, Completion.FinishReason.none)); + } + completionFuture.complete(Completion.FinishReason.stop); + }); + } catch (RejectedExecutionException e) { + // If we have too many requests (active + any waiting in queue), we reject the completion + int activeCount = executor.getActiveCount(); + int queueSize = executor.getQueue().size(); + String error = String.format("Rejected completion due to too many requests, " + + "%d active, %d in queue", activeCount, queueSize); + throw new RejectedExecutionException(error); + } + return completionFuture; + } + +} diff --git a/container-search/src/main/java/ai/vespa/llm/clients/OpenAI.java b/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java index 82e19d47c92..82e19d47c92 100644 --- a/container-search/src/main/java/ai/vespa/llm/clients/OpenAI.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java diff --git a/container-search/src/main/java/ai/vespa/llm/clients/package-info.java b/model-integration/src/main/java/ai/vespa/llm/clients/package-info.java index c360245901c..c360245901c 100644 --- a/container-search/src/main/java/ai/vespa/llm/clients/package-info.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/package-info.java diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java index c97e4781889..9639d9598a8 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java @@ -7,7 +7,6 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -34,7 +33,7 @@ public class ImportedMlModels { /** Create a null imported models */ public ImportedMlModels() { - importedModels = Collections.emptyMap(); + importedModels = Map.of(); } public ImportedMlModels(File modelsDirectory, ExecutorService executor, Collection<MlModelImporter> importers) { @@ -53,7 +52,7 @@ public class ImportedMlModels { skippedModels.put(name, Exceptions.toMessageString(e)); } }); - importedModels = Collections.unmodifiableMap(models); + importedModels = Map.copyOf(models); } /** diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java index e985b6d2956..dbd17d96d4f 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java @@ -8,7 +8,6 @@ import com.yahoo.tensor.evaluation.VariableTensor; import com.yahoo.tensor.functions.Rename; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; public class Argument extends IntermediateOperation { @@ -16,7 +15,7 @@ public class Argument extends IntermediateOperation { private OrderedTensorType standardNamingType; // using standard naming convention: d0, d1, ... public Argument(String modelName, String nodeName, OrderedTensorType type) { - super(modelName, nodeName, Collections.emptyList()); + super(modelName, nodeName, List.of()); this.type = type.rename(vespaName() + "_"); standardNamingType = OrderedTensorType.standardType(type); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java index 91c46d1232e..1d430611c49 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java @@ -7,14 +7,13 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; import java.util.Optional; public class Constant extends IntermediateOperation { public Constant(String modelName, String nodeName, OrderedTensorType type) { - super(modelName, nodeName, Collections.emptyList()); + super(modelName, nodeName, List.of()); this.type = type.rename(vespaName() + "_"); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java index 52de27891cf..d9f4a4b7f4c 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java @@ -53,7 +53,7 @@ public abstract class IntermediateOperation { private final List<String> importWarnings = new ArrayList<>(); private Value constantValue = null; - private List<IntermediateOperation> controlInputs = Collections.emptyList(); + private List<IntermediateOperation> controlInputs = List.of(); protected Function<OrderedTensorType, Value> constantValueFunction = null; @@ -259,7 +259,7 @@ public abstract class IntermediateOperation { if (result == DoubleValue.NaN) { if (constantValue != null) { result = constantValue; - } else if (inputs.size() == 0) { + } else if (inputs.isEmpty()) { if (getConstantValue().isEmpty()) { throw new IllegalArgumentException("Error in evaluating constant for " + name); } @@ -278,7 +278,7 @@ public abstract class IntermediateOperation { /** Insert an operation between an input and this one */ public void insert(IntermediateOperation operationToInsert, int inputNumber) { - if ( operationToInsert.inputs.size() > 0 ) { + if (!operationToInsert.inputs.isEmpty()) { throw new IllegalArgumentException("Operation to insert to '" + name + "' has " + "existing inputs which is not supported."); } @@ -336,7 +336,7 @@ public abstract class IntermediateOperation { public abstract IntermediateOperation withInputs(List<IntermediateOperation> inputs); String asString(Optional<OrderedTensorType> type) { - return type.map(t -> t.toString()).orElse("(unknown)"); + return type.map(OrderedTensorType::toString).orElse("(unknown)"); } /** @@ -373,7 +373,7 @@ public abstract class IntermediateOperation { public String toFullString() { return "\t" + type + ":\t" + operationName() + "(" + - inputs().stream().map(input -> input.toFullString()).collect(Collectors.joining(", ")) + + inputs().stream().map(IntermediateOperation::toFullString).collect(Collectors.joining(", ")) + ")"; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java index ba056d362ac..fa115d8af97 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java @@ -5,13 +5,12 @@ import ai.vespa.rankingexpression.importer.OrderedTensorType; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; public class NoOp extends IntermediateOperation { public NoOp(String modelName, String nodeName, List<IntermediateOperation> inputs) { - super(modelName, nodeName, Collections.emptyList()); // don't propagate inputs + super(modelName, nodeName, List.of()); // don't propagate inputs } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java index dff548cf319..2e48d65fce2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java @@ -9,7 +9,6 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -31,7 +30,7 @@ public class OnnxConstant extends IntermediateOperation { if (value instanceof TensorValue) { type = OrderedTensorType.fromSpec(value.type().toString()).rename(vespaName() + "_"); } else { - type = OrderedTensorType.fromDimensionList(TensorType.Value.DOUBLE, Collections.emptyList()); + type = OrderedTensorType.fromDimensionList(TensorType.Value.DOUBLE, List.of()); } return type; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java index 5a7bbc95889..068c06acd8c 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java @@ -7,7 +7,6 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; /** @@ -20,7 +19,7 @@ public class Rename extends IntermediateOperation { private String from, to; public Rename(String modelName, String from, String to, IntermediateOperation input) { - super(modelName, "rename", input != null ? List.of(input) : Collections.emptyList()); + super(modelName, "rename", input != null ? List.of(input) : List.of()); this.from = from; this.to = to; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java index c4d728e9661..6d0a6c3b04b 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java @@ -10,7 +10,6 @@ import com.yahoo.tensor.functions.ScalarFunctions; import com.yahoo.tensor.functions.TensorFunction; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -77,7 +76,7 @@ public class Softmax extends IntermediateOperation { private class SoftmaxPartialOperation extends IntermediateOperation { private SoftmaxPartialOperation(String modelName, String nodeName, List<IntermediateOperation> inputs) { - super(modelName, nodeName + "_partial" , inputs != null ? inputs : Collections.emptyList()); + super(modelName, nodeName + "_partial" , inputs != null ? inputs : List.of()); } @Override diff --git a/container-search/src/main/resources/configdefinitions/llm-client.def b/model-integration/src/main/resources/configdefinitions/llm-client.def index 0866459166a..0866459166a 100755 --- a/container-search/src/main/resources/configdefinitions/llm-client.def +++ b/model-integration/src/main/resources/configdefinitions/llm-client.def diff --git a/model-integration/src/main/resources/configdefinitions/llm-local-client.def b/model-integration/src/main/resources/configdefinitions/llm-local-client.def new file mode 100755 index 00000000000..4823a53ec46 --- /dev/null +++ b/model-integration/src/main/resources/configdefinitions/llm-local-client.def @@ -0,0 +1,29 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.llm.clients + +# The LLM model to use +model model + +# Maximum number of requests to handle in parallel pr container node +parallelRequests int default=1 + +# Additional number of requests to put in queue for processing before starting to reject new requests +maxQueueSize int default=10 + +# Use GPU +useGpu bool default=true + +# Maximum number of model layers to run on GPU +gpuLayers int default=1000000 + +# Number of threads to use for CPU processing - -1 means use all available cores +# Not used for GPU processing +threads int default=-1 + +# Context size for the model +# Context is divided between parallel requests. So for 10 parallel requests, each "slot" gets 1/10 of the context +contextSize int default=4096 + +# Maximum number of tokens to process in one request - overriden by inference parameters +maxTokens int default=512 + diff --git a/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java index be75c4d3351..f6216e4149c 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/ColBertEmbedderTest.java @@ -61,27 +61,94 @@ public class ColBertEmbedderTest { TensorType.fromSpec("tensor<int8>(dt{},x[2])"), "tensor<int8>(dt{},x[2]):{0:[1.0, -128.0], 1:[5.0, 1.0]}",2 ); + assertPackedRight( + "" + + "tensor<float>(d0[1],d1[2],d2[16]):" + + "[[" + + "[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0]," + + "[0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1]" + + "]]", + TensorType.fromSpec("tensor<int8>(dt{},x[1])"), + "tensor<int8>(dt{},x[1]):{0:1.0, 1:5.0}",2 + ); + } + + @Test + public void testCachingFloat() { + int initialEmbeddingsDone = runtime.embeddingsDone; + var context = new Embedder.Context("schema.indexing"); + + var input = "This is a test string to embed"; + var t1 = (MixedTensor) embedder.embed(input, context,TensorType.fromSpec("tensor<float>(dt{},x[8])")); + assertEquals(initialEmbeddingsDone + 1, runtime.embeddingsDone); + + var t2 = (MixedTensor)embedder.embed(input, context,TensorType.fromSpec("tensor<float>(dt{},x[4])")); + assertEquals("Cached value was used", initialEmbeddingsDone + 1, runtime.embeddingsDone); + + assertNotEquals(t1,t2); + for(int token = 0; token < 7; token ++) { + for(int dim = 0; dim < 4; dim++) { // the four first should be equal + assertEquals(t1.get(TensorAddress.of(token,dim)),t2.get(TensorAddress.of(token,dim)), 1e-6); + } + } + // t2 only has 4 dimensions so this should be out of bounds which returns 0 + assertEquals(0, t2.get(TensorAddress.of(1,4)), 1e-6); + + input = "This is a different test string to embed"; + embedder.embed(input, context,TensorType.fromSpec("tensor<float>(dt{},x[8])")); + assertEquals(initialEmbeddingsDone + 2, runtime.embeddingsDone); + } + + @Test + public void testCachingInt() { + int initialEmbeddingsDone = runtime.embeddingsDone; + var context = new Embedder.Context("schema.indexing"); + + var input = "This is a test string to embed"; + var t1 = (MixedTensor) embedder.embed(input, context, TensorType.fromSpec("tensor<int8>(dt{},x[8])")); + assertEquals(initialEmbeddingsDone + 1, runtime.embeddingsDone); + + var t2 = (MixedTensor)embedder.embed(input, context, TensorType.fromSpec("tensor<int8>(dt{},x[4])")); + assertEquals("Cached value was used", initialEmbeddingsDone + 1, runtime.embeddingsDone); + + assertNotEquals(t1, t2); + for(int token = 0; token < 7; token ++) { + for(int dim = 0; dim < 4; dim++) { // the four first should be equal + assertEquals(t1.get(TensorAddress.of(token,dim)), t2.get(TensorAddress.of(token,dim)), 1e-6); + } + } + // t2 only has 4 dimensions so this should be out of bounds which returns 0 + assertEquals(0, t2.get(TensorAddress.of(0,4)), 1e-6); + + input = "This is a different test string to embed"; + embedder.embed(input, context,TensorType.fromSpec("tensor<float>(dt{},x[8])")); + assertEquals(initialEmbeddingsDone + 2, runtime.embeddingsDone); } + @Test public void testEmbedder() { - assertEmbed("tensor<float>(dt{},x[128])", "this is a document", indexingContext); - assertEmbed("tensor<int8>(dt{},x[16])", "this is a document", indexingContext); - assertEmbed("tensor<float>(qt{},x[128])", "this is a query", queryContext); + var indexingContext = new Embedder.Context("schema.indexing"); + assertEmbed("tensor<float>(dt{},x[128])", "this is a document", indexingContext,128); + assertEmbed("tensor<float>(dt{},x[64])", "this is a document", indexingContext,64); - assertThrows(IllegalArgumentException.class, () -> { - // throws because int8 is not supported for query context - assertEmbed("tensor<int8>(qt{},x[16])", "this is a query", queryContext); - }); + assertEmbed("tensor<int8>(dt{},x[16])", "this is a document", indexingContext,16); + assertEmbed("tensor<int8>(dt{},x[8])", "this is a document", indexingContext,8); + assertEmbed("tensor<int8>(dt{},x[4])", "this is a document", indexingContext,4); + assertEmbed("tensor<int8>(dt{},x[3])", "this is a document", indexingContext,3); + + var queryContext = new Embedder.Context("query(qt{})"); + assertEmbed("tensor<float>(qt{},x[128])", "this is a query", queryContext,128); + assertEmbed("tensor<float>(qt{},x[64])", "this is a query", queryContext,64); assertThrows(IllegalArgumentException.class, () -> { - // throws because 16 is less than model output (128) and we want float - assertEmbed("tensor<float>(qt{},x[16])", "this is a query", queryContext); + // throws because int8 is not supported for query context + assertEmbed("tensor<int8>(qt{},x[16])", "this is a query", queryContext,16); }); assertThrows(IllegalArgumentException.class, () -> { - // throws because 128/8 does not fit into 15 - assertEmbed("tensor<int8>(qt{},x[15])", "this is a query", indexingContext); + // throws because 8*32 is larger than (128) + assertEmbed("tensor<int8>(qt{},x[32])", "this is a query", queryContext,32); }); } @@ -130,26 +197,32 @@ public class ColBertEmbedderTest { } @Test - public void testLenghtLimits() { + public void testLengthLimits() { StringBuilder sb = new StringBuilder(); for(int i = 0; i < 1024; i++) { sb.append("annoyance"); sb.append(" "); } String text = sb.toString(); - Tensor fullFloat = assertEmbed("tensor<float>(dt{},x[128])", text, indexingContext); - assertEquals(512*128,fullFloat.size()); + var indexingContext = new Embedder.Context("schema.indexing"); - Tensor query = assertEmbed("tensor<float>(dt{},x[128])", text, queryContext); - assertEquals(32*128,query.size()); + Tensor fullFloat = assertEmbed("tensor<float>(dt{},x[128])", text, indexingContext,128); + assertEquals(512*128,fullFloat.size()); - Tensor binaryRep = assertEmbed("tensor<int8>(dt{},x[16])", text, indexingContext); + Tensor binaryRep = assertEmbed("tensor<int8>(dt{},x[16])", text, indexingContext,16); assertEquals(512*16,binaryRep.size()); - Tensor shortDoc = assertEmbed("tensor<int8>(dt{},x[16])", "annoyance", indexingContext); + Tensor shortDoc = assertEmbed("tensor<int8>(dt{},x[16])", "annoyance", indexingContext,16); // 4 tokens, 16 bytes each = 64 bytes //CLS [unused1] sequence assertEquals(4*16,shortDoc.size());; + + var queryContext = new Embedder.Context("query(qt{})"); + Tensor query = assertEmbed("tensor<float>(dt{},x[128])", text, queryContext,128); + assertEquals(32*128,query.size()); + + Tensor shortQuery = assertEmbed("tensor<float>(dt{},x[64])", text, queryContext,64); + assertEquals(32*64,shortQuery.size()); } @Ignore @@ -163,18 +236,19 @@ public class ColBertEmbedderTest { long now = System.currentTimeMillis(); int n = 1000; for (int i = 0; i < n; i++) { - assertEmbed("tensor<float>(dt{},x[128])", text, indexingContext); + assertEmbed("tensor<float>(dt{},x[128])", text, new Embedder.Context("schema.indexing"),128); } long elapsed = (System.currentTimeMillis() - now); System.out.println("Elapsed time: " + elapsed + " ms"); } - static Tensor assertEmbed(String tensorSpec, String text, Embedder.Context context) { + static Tensor assertEmbed(String tensorSpec, String text, Embedder.Context context, int dimSize) { TensorType destType = TensorType.fromSpec(tensorSpec); Tensor result = embedder.embed(text, context, destType); assertEquals(destType,result.type()); MixedTensor mixedTensor = (MixedTensor) result; - if (context == queryContext) { + assertEquals(dimSize,mixedTensor.denseSubspaceSize()); + if (context.getDestination().startsWith("query")) { assertEquals(32*mixedTensor.denseSubspaceSize(),mixedTensor.size()); } return result; @@ -182,12 +256,12 @@ public class ColBertEmbedderTest { static void assertPackedRight(String numbers, TensorType destination, String expected, int size) { var in = (IndexedTensor) Tensor.from(numbers); + int targetDim = destination.indexedSubtype().dimensions().get(0).size().get().intValue(); Tensor packed = ColBertEmbedder.toBitTensor(in, destination, size); assertEquals(expected, packed.toString()); Tensor unpacked = ColBertEmbedder.expandBitTensor(packed); - assertEquals(in.shape()[2], unpacked.type().indexedSubtype().dimensions().get(0).size().get().longValue()); for (int dOuter = 0; dOuter < size; dOuter++) { - for (int dInner = 0; dInner < in.shape()[2]; dInner++) { + for (int dInner = 0; dInner < targetDim*8; dInner++) { var addr = TensorAddress.of(dOuter, dInner); double oldVal = in.get(TensorAddress.of(0,dOuter, dInner)); if (oldVal > 0) { @@ -200,19 +274,16 @@ public class ColBertEmbedderTest { } static final ColBertEmbedder embedder; - static final ColBertEmbedder multiLingualEmbedder; - static final Embedder.Context indexingContext; - static final Embedder.Context queryContext; + static final CountingRuntime runtime; static { - indexingContext = new Embedder.Context("schema.indexing"); - queryContext = new Embedder.Context("query(qt)"); - embedder = getEmbedder(); - multiLingualEmbedder = getMultiLingualEmbedder(); + runtime = new CountingRuntime(); + embedder = createEmbedder(runtime); + multiLingualEmbedder = getMultiLingualEmbedder(runtime); } - private static ColBertEmbedder getEmbedder() { + private static ColBertEmbedder createEmbedder(Embedder.Runtime runtime) { String vocabPath = "src/test/models/onnx/transformer/real_tokenizer.json"; String modelPath = "src/test/models/onnx/transformer/colbert-dummy-v2.onnx"; assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); @@ -220,10 +291,10 @@ public class ColBertEmbedderTest { builder.tokenizerPath(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); builder.transformerGpuDevice(-1); - return new ColBertEmbedder(new OnnxRuntime(), Embedder.Runtime.testInstance(), builder.build()); + return new ColBertEmbedder(new OnnxRuntime(), runtime, builder.build()); } - private static ColBertEmbedder getMultiLingualEmbedder() { + private static ColBertEmbedder getMultiLingualEmbedder(Embedder.Runtime runtime) { String vocabPath = "src/test/models/onnx/transformer/sentence_piece_tokenizer.json"; String modelPath = "src/test/models/onnx/transformer/colbert-dummy-v2.onnx"; assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); @@ -239,7 +310,21 @@ public class ColBertEmbedderTest { builder.queryTokenId(3); builder.documentTokenId(4); - return new ColBertEmbedder(new OnnxRuntime(), Embedder.Runtime.testInstance(), builder.build()); + return new ColBertEmbedder(new OnnxRuntime(), Embedder.Runtime.testInstance(), builder.build()); + } + + private static class CountingRuntime implements Embedder.Runtime { + + int embeddingsDone = 0; + + @Override + public void sampleEmbeddingLatency(double millis, Embedder.Context ctx) { + embeddingsDone++; + } + + @Override + public void sampleSequenceLength(long length, Embedder.Context ctx) { } + } } diff --git a/model-integration/src/test/java/ai/vespa/embedding/HuggingFaceEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java index 1ce1d955b00..d504d77cc9b 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/HuggingFaceEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.embedding; +package ai.vespa.embedding.huggingface; + -import ai.vespa.embedding.huggingface.HuggingFaceEmbedder; import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.ModelReference; import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; @@ -12,10 +12,10 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.TensorAddress; import org.junit.Test; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; @@ -26,7 +26,6 @@ public class HuggingFaceEmbedderTest { static HuggingFaceEmbedder embedder = getEmbedder(); static HuggingFaceEmbedder normalizedEmbedder = getNormalizedEmbedder(); - static Embedder.Context context = new Embedder.Context("schema.indexing"); @Test public void testBinarization() { @@ -48,16 +47,48 @@ public class HuggingFaceEmbedderTest { private void assertPackRight(String input, String expected, TensorType type) { Tensor inputTensor = Tensor.from(input); Tensor result = HuggingFaceEmbedder.binarize((IndexedTensor) inputTensor, type); - assertEquals(expected.toString(), result.toString()); - //Verify against what is done in ranking with unpack_bits + assertEquals(expected, result.toString()); + //Verify that the unpack_bits ranking feature produce compatible output Tensor unpacked = expandBitTensor(result); assertEquals(inputTensor.toString(), unpacked.toString()); } @Test + public void testCaching() { + var context = new Embedder.Context("schema.indexing"); + var myEmbedderId = "my-hf-embedder"; + context.setEmbedderId(myEmbedderId); + + var input = "This is a test string to embed"; + Tensor result = embedder.embed(input, context,TensorType.fromSpec("tensor<float>(x[8])")); + HuggingFaceEmbedder.HFEmbedderCacheKey key = new HuggingFaceEmbedder.HFEmbedderCacheKey(myEmbedderId, input); + var modelOuput = context.getCachedValue(key); + assertNotNull(modelOuput); + + Tensor binaryResult = embedder.embed(input, context,TensorType.fromSpec("tensor<int8>(x[4])")); + var modelOuput2 = context.getCachedValue(key); + assertEquals(modelOuput, modelOuput2); + assertNotEquals(result, binaryResult); + + var anotherInput = "This is a different test string to embed with the same embedder"; + embedder.embed(anotherInput, context,TensorType.fromSpec("tensor<float>(x[4])")); + key = new HuggingFaceEmbedder.HFEmbedderCacheKey(myEmbedderId, anotherInput); + var modelOuput3 = context.getCachedValue(key); + assertNotEquals(modelOuput, modelOuput3); + + //context cache is shared + var copyContext = context.copy(); + var anotherEmbedderId = "another-hf-embedder"; + copyContext.setEmbedderId(anotherEmbedderId); + key = new HuggingFaceEmbedder.HFEmbedderCacheKey(anotherEmbedderId, input); + assertNull(copyContext.getCachedValue(key)); + embedder.embed(input, copyContext,TensorType.fromSpec("tensor<int8>(x[2])")); + assertNotEquals(modelOuput, copyContext.getCachedValue(key)); + } + @Test public void testEmbedder() { + var context = new Embedder.Context("schema.indexing"); String input = "This is a test"; - Tensor expected = Tensor.from("tensor<float>(x[8]):[-0.666, 0.335, 0.227, 0.0919, -0.069, 0.323, 0.422, 0.270]"); Tensor result = embedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x[8])"))); for(int i = 0; i < 8; i++) { @@ -85,16 +116,33 @@ public class HuggingFaceEmbedderTest { @Test public void testEmbedderWithNormalization() { String input = "This is a test"; - + var context = new Embedder.Context("schema.indexing"); Tensor result = normalizedEmbedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x[8])"))); assertEquals(1.0, result.multiply(result).sum().asDouble(), 1e-3); - result = normalizedEmbedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x[16])"))); assertEquals(1.0, result.multiply(result).sum().asDouble(), 1e-3); Tensor binarizedResult = embedder.embed(input, context, TensorType.fromSpec(("tensor<int8>(x[2])"))); assertEquals("tensor<int8>(x[2]):[119, 44]", binarizedResult.toAbbreviatedString()); } + @Test + public void testThatWrongTensorTypeThrows() { + var context = new Embedder.Context("schema.indexing"); + String input = "This is a test"; + assertThrows(IllegalArgumentException.class, () -> { + // throws because the target tensor type is mapped + embedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x{})"))); + }); + assertThrows(IllegalArgumentException.class, () -> { + // throws because the target tensor is 0d + embedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x[0]"))); + }); + assertThrows(IllegalArgumentException.class, () -> { + // throws because the target tensor is 2d + embedder.embed(input, context, TensorType.fromSpec(("tensor<float>(x{}, y[2])"))); + }); + } + private static HuggingFaceEmbedder getEmbedder() { String vocabPath = "src/test/models/onnx/transformer/real_tokenizer.json"; String modelPath = "src/test/models/onnx/transformer/embedding_model.onnx"; diff --git a/container-search/src/test/java/ai/vespa/llm/clients/ConfigurableLanguageModelTest.java b/model-integration/src/test/java/ai/vespa/llm/clients/ConfigurableLanguageModelTest.java index a9f4c3dfac5..35d5cfd3855 100644 --- a/container-search/src/test/java/ai/vespa/llm/clients/ConfigurableLanguageModelTest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/ConfigurableLanguageModelTest.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.Collections; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -85,7 +84,7 @@ public class ConfigurableLanguageModelTest { } private static InferenceParameters inferenceParams() { - return new InferenceParameters(s -> lookupParameter(s, Collections.emptyMap())); + return new InferenceParameters(s -> lookupParameter(s, Map.of())); } private static InferenceParameters inferenceParams(Map<String, String> params) { @@ -93,7 +92,7 @@ public class ConfigurableLanguageModelTest { } private static InferenceParameters inferenceParamsWithDefaultKey() { - return new InferenceParameters(MockLLMClient.ACCEPTED_API_KEY, s -> lookupParameter(s, Collections.emptyMap())); + return new InferenceParameters(MockLLMClient.ACCEPTED_API_KEY, s -> lookupParameter(s, Map.of())); } private LlmClientConfig modelParams(String apiKeySecretName, String endpoint) { diff --git a/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java new file mode 100644 index 00000000000..a3b260f3fb5 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/llm/clients/LocalLLMTest.java @@ -0,0 +1,186 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.llm.clients; + +import ai.vespa.llm.InferenceParameters; +import ai.vespa.llm.completion.Completion; +import ai.vespa.llm.completion.Prompt; +import ai.vespa.llm.completion.StringPrompt; +import com.yahoo.config.ModelReference; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for LocalLLM. + * + * @author lesters + */ +public class LocalLLMTest { + + private static String model = "src/test/models/llm/tinyllm.gguf"; + private static Prompt prompt = StringPrompt.from("A random prompt"); + + @Test + @Disabled + public void testGeneration() { + var config = new LlmLocalClientConfig.Builder() + .parallelRequests(1) + .model(ModelReference.valueOf(model)); + var llm = new LocalLLM(config.build()); + + try { + var result = llm.complete(prompt, defaultOptions()); + assertEquals(Completion.FinishReason.stop, result.get(0).finishReason()); + assertTrue(result.get(0).text().length() > 10); + } finally { + llm.deconstruct(); + } + } + + @Test + @Disabled + public void testAsyncGeneration() { + var sb = new StringBuilder(); + var tokenCount = new AtomicInteger(0); + var config = new LlmLocalClientConfig.Builder() + .parallelRequests(1) + .model(ModelReference.valueOf(model)); + var llm = new LocalLLM(config.build()); + + try { + var future = llm.completeAsync(prompt, defaultOptions(), completion -> { + sb.append(completion.text()); + tokenCount.incrementAndGet(); + }).exceptionally(exception -> Completion.FinishReason.error); + + assertFalse(future.isDone()); + var reason = future.join(); + assertTrue(future.isDone()); + assertNotEquals(reason, Completion.FinishReason.error); + + } finally { + llm.deconstruct(); + } + assertTrue(tokenCount.get() > 0); + System.out.println(sb); + } + + @Test + @Disabled + public void testParallelGeneration() { + var prompts = testPrompts(); + var promptsToUse = prompts.size(); + var parallelRequests = 10; + + var futures = new ArrayList<CompletableFuture<Completion.FinishReason>>(Collections.nCopies(promptsToUse, null)); + var completions = new ArrayList<StringBuilder>(Collections.nCopies(promptsToUse, null)); + var tokenCounts = new ArrayList<>(Collections.nCopies(promptsToUse, 0)); + + var config = new LlmLocalClientConfig.Builder() + .parallelRequests(parallelRequests) + .model(ModelReference.valueOf(model)); + var llm = new LocalLLM(config.build()); + + try { + for (int i = 0; i < promptsToUse; i++) { + final var seq = i; + + completions.set(seq, new StringBuilder()); + futures.set(seq, llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { + completions.get(seq).append(completion.text()); + tokenCounts.set(seq, tokenCounts.get(seq) + 1); + }).exceptionally(exception -> Completion.FinishReason.error)); + } + for (int i = 0; i < promptsToUse; i++) { + var reason = futures.get(i).join(); + assertNotEquals(reason, Completion.FinishReason.error); + } + } finally { + llm.deconstruct(); + } + for (int i = 0; i < promptsToUse; i++) { + assertFalse(completions.get(i).isEmpty()); + assertTrue(tokenCounts.get(i) > 0); + } + } + + @Test + @Disabled + public void testRejection() { + var prompts = testPrompts(); + var promptsToUse = prompts.size(); + var parallelRequests = 2; + var additionalQueue = 1; + // 7 should be rejected + + var futures = new ArrayList<CompletableFuture<Completion.FinishReason>>(Collections.nCopies(promptsToUse, null)); + var completions = new ArrayList<StringBuilder>(Collections.nCopies(promptsToUse, null)); + + var config = new LlmLocalClientConfig.Builder() + .parallelRequests(parallelRequests) + .maxQueueSize(additionalQueue) + .model(ModelReference.valueOf(model)); + var llm = new LocalLLM(config.build()); + + var rejected = new AtomicInteger(0); + try { + for (int i = 0; i < promptsToUse; i++) { + final var seq = i; + + completions.set(seq, new StringBuilder()); + try { + var future = llm.completeAsync(StringPrompt.from(prompts.get(seq)), defaultOptions(), completion -> { + completions.get(seq).append(completion.text()); + }).exceptionally(exception -> Completion.FinishReason.error); + futures.set(seq, future); + } catch (RejectedExecutionException e) { + rejected.incrementAndGet(); + } + } + for (int i = 0; i < promptsToUse; i++) { + if (futures.get(i) != null) { + assertNotEquals(futures.get(i).join(), Completion.FinishReason.error); + } + } + } finally { + llm.deconstruct(); + } + assertEquals(7, rejected.get()); + } + + private static InferenceParameters defaultOptions() { + final Map<String, String> options = Map.of( + "temperature", "0.1", + "npredict", "100" + ); + return new InferenceParameters(options::get); + } + + private List<String> testPrompts() { + List<String> prompts = new ArrayList<>(); + prompts.add("Write a short story about a time-traveling detective who must solve a mystery that spans multiple centuries."); + prompts.add("Explain the concept of blockchain technology and its implications for data security in layman's terms."); + prompts.add("Discuss the socio-economic impacts of the Industrial Revolution in 19th century Europe."); + prompts.add("Describe a future where humans have colonized Mars, focusing on daily life and societal structure."); + prompts.add("Analyze the statement 'If a tree falls in a forest and no one is around to hear it, does it make a sound?' from both a philosophical and a physics perspective."); + prompts.add("Translate the following sentence into French: 'The quick brown fox jumps over the lazy dog.'"); + prompts.add("Explain what the following Python code does: `print([x for x in range(10) if x % 2 == 0])`."); + prompts.add("Provide general guidelines for maintaining a healthy lifestyle to reduce the risk of developing heart disease."); + prompts.add("Create a detailed description of a fictional planet, including its ecosystem, dominant species, and technology level."); + prompts.add("Discuss the impact of social media on interpersonal communication in the 21st century."); + return prompts; + } + +} diff --git a/container-search/src/test/java/ai/vespa/llm/clients/MockLLMClient.java b/model-integration/src/test/java/ai/vespa/llm/clients/MockLLMClient.java index 4d0073f1cbe..4d0073f1cbe 100644 --- a/container-search/src/test/java/ai/vespa/llm/clients/MockLLMClient.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/MockLLMClient.java diff --git a/container-search/src/test/java/ai/vespa/llm/clients/OpenAITest.java b/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java index 57339f6ad49..57339f6ad49 100644 --- a/container-search/src/test/java/ai/vespa/llm/clients/OpenAITest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java diff --git a/model-integration/src/test/models/llm/tinyllm.gguf b/model-integration/src/test/models/llm/tinyllm.gguf Binary files differnew file mode 100644 index 00000000000..34367b6b57b --- /dev/null +++ b/model-integration/src/test/models/llm/tinyllm.gguf diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 8646121bd4b..f00414aa654 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.Capacity; @@ -33,6 +34,7 @@ public class Cluster { private final ClusterResources min, max; private final IntRange groupSize; private final boolean required; + private final Optional<CloudAccount> cloudAccount; private final List<Autoscaling> suggestions; private final Autoscaling target; private final ClusterInfo clusterInfo; @@ -47,6 +49,7 @@ public class Cluster { ClusterResources maxResources, IntRange groupSize, boolean required, + Optional<CloudAccount> cloudAccount, List<Autoscaling> suggestions, Autoscaling target, ClusterInfo clusterInfo, @@ -58,6 +61,7 @@ public class Cluster { this.max = Objects.requireNonNull(maxResources); this.groupSize = Objects.requireNonNull(groupSize); this.required = required; + this.cloudAccount = Objects.requireNonNull(cloudAccount); this.suggestions = Objects.requireNonNull(suggestions); Objects.requireNonNull(target); if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources)) @@ -89,6 +93,9 @@ public class Cluster { */ public boolean required() { return required; } + /** Returns the enclave cloud account of this cluster, or empty if not enclave. */ + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + /** * Returns the computed resources (between min and max, inclusive) this cluster should * have allocated at the moment (whether or not it actually has it), @@ -134,19 +141,19 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); + capacity.cloudAccount(), suggestions, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggestions(List<Autoscaling> suggestions) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, cloudAccount, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, cloudAccount, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, cloudAccount, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -160,7 +167,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, cloudAccount, suggestions, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -192,7 +199,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); + requested.cloudAccount(), List.of(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index cb70eb977c4..75a00fa951e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -35,10 +36,12 @@ public class AllocatableResources { /** Fake allocatable resources from requested capacity */ public AllocatableResources(ClusterResources requested, ClusterSpec clusterSpec, - NodeRepository nodeRepository) { + NodeRepository nodeRepository, + CloudAccount cloudAccount) { this.nodes = requested.nodes(); this.groups = requested.groups(); - this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec), false); + this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), cloudAccount, + nodeRepository.exclusiveAllocation(clusterSpec), false); this.advertisedResources = requested.nodeResources(); this.clusterSpec = clusterSpec; this.fulfilment = 1; @@ -180,17 +183,20 @@ public class AllocatableResources { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources var allocatableResources = calculateAllocatableResources(wantedResources, nodeRepository, + model.cloudAccount(), clusterSpec, applicationLimits, exclusive, true); var worstCaseRealResources = nodeRepository.resourcesCalculator().requestToReal(allocatableResources.advertisedResources, + model.cloudAccount(), exclusive, false); if ( ! systemLimits.isWithinRealLimits(worstCaseRealResources, clusterSpec)) { allocatableResources = calculateAllocatableResources(wantedResources, nodeRepository, + model.cloudAccount(), clusterSpec, applicationLimits, exclusive, @@ -210,7 +216,7 @@ public class AllocatableResources { for (Flavor flavor : nodeRepository.flavors().getFlavors()) { // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); - NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, false); + NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, model.cloudAccount(), exclusive, false); // Adjust where we don't need exact match to the flavor if (flavor.resources().storageType() == NodeResources.StorageType.remote) { @@ -251,20 +257,21 @@ public class AllocatableResources { private static AllocatableResources calculateAllocatableResources(ClusterResources wantedResources, NodeRepository nodeRepository, + CloudAccount cloudAccount, ClusterSpec clusterSpec, Limits applicationLimits, boolean exclusive, boolean bestCase) { var systemLimits = nodeRepository.nodeResourceLimits(); - var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive, bestCase); + var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), cloudAccount, exclusive, bestCase); advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterSpec, exclusive, true); // Ask for something legal advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail - var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); // What we'll really get + var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, cloudAccount, exclusive, bestCase); // What we'll really get if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec) && advertisedResources.storageType() == NodeResources.StorageType.any) { // Since local disk reserves some of the storage, try to constrain to remote disk advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); - realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); + realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, cloudAccount, exclusive, bestCase); } return new AllocatableResources(wantedResources.with(realResources), advertisedResources, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 45ef2d1d7b5..61d4ced1367 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.Autoscaler.headroomRequiredToScaleDown; @@ -38,9 +37,7 @@ public class AllocationOptimizer { * @return the best allocation, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits */ - public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment, - ClusterModel model, - Limits limits) { + public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment, ClusterModel model, Limits limits) { return findBestAllocations(loadAdjustment, model, limits).stream().findFirst(); } @@ -51,9 +48,7 @@ public class AllocationOptimizer { * @return the best allocations, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits. The list contains the three best allocations, sorted from most to least preferred. */ - public List<AllocatableResources> findBestAllocations(Load loadAdjustment, - ClusterModel model, - Limits limits) { + public List<AllocatableResources> findBestAllocations(Load loadAdjustment, ClusterModel model, Limits limits) { if (limits.isEmpty()) limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 986ab830283..a0f9d6e260a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -123,6 +124,7 @@ public class ClusterModel { public Application application() { return application; } public ClusterSpec clusterSpec() { return clusterSpec; } + public CloudAccount cloudAccount() { return cluster.cloudAccount().orElse(CloudAccount.empty); } public AllocatableResources current() { return current; } private ClusterNodesTimeseries nodeTimeseries() { return nodeTimeseries; } private ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } @@ -438,6 +440,7 @@ public class ClusterModel { clusterSpec, application.id()); return nodeRepository.resourcesCalculator().requestToReal(initialResources, + cloudAccount(), nodeRepository.exclusiveAllocation(clusterSpec), false).memoryGb(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java index 940109bab8a..8b2dc44669f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MemoryMetricsDb.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; @@ -33,6 +34,11 @@ public class MemoryMetricsDb implements MetricsDb { /** Lock all access for now since we modify lists inside a map */ private final Object lock = new Object(); + @Inject + public MemoryMetricsDb() { + this(Clock.systemUTC()); + } + public MemoryMetricsDb(Clock clock) { this.clock = clock; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index e9230d2c91a..057430b381f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -296,6 +296,11 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } } + private static String getStr(Record record, int col) { + CharSequence charSequence = record.getStrA(col); + return charSequence != null ? charSequence.toString() : ""; + } + private ListMap<String, NodeMetricSnapshot> getNodeSnapshots(Instant startTime, Set<String> hostnames, SqlExecutionContext context) throws SqlException { @@ -312,7 +317,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { try (RecordCursor cursor = factory.getCursor(context)) { Record record = cursor.getRecord(); while (cursor.hasNext()) { - String hostname = record.getStr(0).toString(); + String hostname = getStr(record, 0); if (hostnames.isEmpty() || hostnames.contains(hostname)) { snapshots.put(hostname, new NodeMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(1) / 1000), @@ -345,9 +350,9 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { try (RecordCursor cursor = factory.getCursor(context)) { Record record = cursor.getRecord(); while (cursor.hasNext()) { - String applicationIdString = record.getStr(0).toString(); + String applicationIdString = getStr(record, 0); if ( ! application.serializedForm().equals(applicationIdString)) continue; - String clusterId = record.getStr(1).toString(); + String clusterId = getStr(record, 1); if (cluster.value().equals(clusterId)) { snapshots.add(new ClusterMetricSnapshot(Instant.ofEpochMilli(record.getTimestamp(2) / 1000), record.getFloat(3), @@ -475,7 +480,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { try (RecordCursor cursor = factory.getCursor(context)) { Record record = cursor.getRecord(); while (cursor.hasNext()) { - columns.add(record.getStr(0).toString()); + columns.add(getStr(record, 0)); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index ab103b0bfcf..939c958efdd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -27,8 +27,8 @@ public class DirtyExpirer extends Expirer { private final boolean wantToDeprovisionOnExpiry; - DirtyExpirer(NodeRepository nodeRepository, Duration dirtyTimeout, Metric metric) { - super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, dirtyTimeout, metric); + DirtyExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) { + super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, expiryTime, metric); // Deprovision hosts on expiry if dynamically provisioned this.wantToDeprovisionOnExpiry = nodeRepository.zone().cloud().dynamicProvisioning(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index d97f4566e57..76abf5c2aef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -16,7 +16,7 @@ import java.util.List; /** * Maintenance job which moves inactive nodes to dirty or parked after timeout. * - * The timeout is in place to provide a grace period in which nodes can be brought back to active + * The expiry time is in place to provide a grace period in which nodes can be brought back to active * if they were deactivated in error. As inactive nodes retain their state * they can be brought back to active and correct state faster than a new node. * @@ -32,12 +32,12 @@ import java.util.List; public class InactiveExpirer extends Expirer { private final NodeRepository nodeRepository; - private final Duration timeout; + private final Duration expiryTime; - InactiveExpirer(NodeRepository nodeRepository, Duration timeout, Metric metric) { - super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, timeout, metric); + InactiveExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) { + super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, expiryTime, metric); this.nodeRepository = nodeRepository; - this.timeout = timeout; + this.expiryTime = expiryTime; } @Override @@ -49,12 +49,12 @@ public class InactiveExpirer extends Expirer { @Override protected boolean isExpired(Node node) { - return super.isExpired(node, timeout(node)) || + return super.isExpired(node, expiryTime(node)) || node.allocation().get().owner().instance().isTester(); } - private Duration timeout(Node node) { - return timeout; + private Duration expiryTime(Node node) { + return expiryTime; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index 24901cb10a9..adcc486d5b4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -27,8 +27,8 @@ public class ProvisionedExpirer extends Expirer { private final NodeRepository nodeRepository; - ProvisionedExpirer(NodeRepository nodeRepository, Duration timeout, Metric metric) { - super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, timeout, metric); + ProvisionedExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) { + super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, expiryTime, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java index 8e82307deb1..e10a4e82a0c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java @@ -6,7 +6,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -20,8 +19,8 @@ public class Reports { private final Map<String, Report> reports; - public Reports() { this(Collections.emptyMap()); } - private Reports(Map<String, Report> reports) { this.reports = Collections.unmodifiableMap(reports); } + public Reports() { this(Map.of()); } + private Reports(Map<String, Report> reports) { this.reports = Map.copyOf(reports); } public boolean isEmpty() { return reports.isEmpty(); } public Optional<Report> getReport(String id) { return Optional.ofNullable(reports.get(id)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java index d5aa82a8dc2..821fcd51e18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/ParentHostFilter.java @@ -12,7 +12,7 @@ import java.util.stream.Collectors; /** * Filter based on the parent host value (for virtualized nodes). * - * @author dybis + * @author Haakon Dybdahl */ public class ParentHostFilter { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 1315207efb8..8a5780496b1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; @@ -56,7 +57,7 @@ public class ApplicationSerializer { private static final String maxResourcesKey = "max"; private static final String groupSizeKey = "groupSize"; private static final String requiredKey = "required"; - private static final String suggestedKey = "suggested"; + private static final String cloudAccountKey = "cloudAccount"; private static final String suggestionsKey = "suggestionsKey"; private static final String clusterInfoKey = "clusterInfo"; private static final String bcpDeadlineKey = "bcpDeadline"; @@ -141,8 +142,7 @@ public class ApplicationSerializer { toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); toSlime(cluster.groupSize(), clusterObject.setObject(groupSizeKey)); clusterObject.setBool(requiredKey, cluster.required()); - // TODO(olaa): Remove 'suggested' once all configservers have stopped reading entry - toSlime(Autoscaling.empty(), clusterObject.setObject(suggestedKey)); + cluster.cloudAccount().ifPresent(cloudAccount -> clusterObject.setString(cloudAccountKey, cloudAccount.value())); toSlime(cluster.suggestions(), clusterObject.setArray(suggestionsKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); if (! cluster.clusterInfo().isEmpty()) @@ -159,6 +159,7 @@ public class ApplicationSerializer { clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), intRangeFromSlime(clusterObject.field(groupSizeKey)), clusterObject.field(requiredKey).asBool(), + optionalCloudAccount(clusterObject.field(cloudAccountKey)), suggestionsFromSlime(clusterObject.field(suggestionsKey)), autoscalingFromSlime(clusterObject.field(targetKey)), clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), @@ -329,6 +330,10 @@ public class ApplicationSerializer { }; } + private static Optional<CloudAccount> optionalCloudAccount(Inspector inspector) { + return inspector.valid() ? Optional.of(CloudAccount.from(inspector.asString())) : Optional.empty(); + } + private static Optional<Instant> optionalInstant(Inspector inspector) { return inspector.valid() ? Optional.of(Instant.ofEpochMilli(inspector.asLong())) : Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index d511570881b..d2edaaf3737 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -157,7 +157,7 @@ public class CuratorDb { * @return the nodes in their persisted state */ public List<Node> writeTo(List<Node> nodes, Agent agent, Optional<String> reason) { - if (nodes.isEmpty()) return Collections.emptyList(); + if (nodes.isEmpty()) return List.of(); List<Node> writtenNodes = new ArrayList<>(nodes.size()); @@ -191,7 +191,7 @@ public class CuratorDb { } public Node writeTo(Node.State toState, Node node, Agent agent, Optional<String> reason) { - return writeTo(toState, Collections.singletonList(node), agent, reason).get(0); + return writeTo(toState, List.of(node), agent, reason).get(0); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index a1306d7831a..ec48e1d5f34 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -51,6 +51,8 @@ public class CapacityPolicies { private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application, boolean exclusive) { int nodes = decideSize(resources.nodes(), capacity.isRequired(), application.instance().isTester()); int groups = Math.min(resources.groups(), nodes); // cannot have more groups than nodes + while (groups > 1 && nodes % groups != 0) + groups--; // Must be divisible by the number of groups var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired(), exclusive); return new ClusterResources(nodes, groups, nodeResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 0d6a98f50a3..dae4b11a609 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -41,10 +42,10 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { return resources; } + public NodeResources requestToReal(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources; } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { return resources; } + public NodeResources realToRequest(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources; } @Override public long reservedDiskSpaceInBase2Gb(NodeType nodeType, boolean sharedHost) { return 0; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index e474ae6eea3..204660f9869 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -28,13 +29,14 @@ public interface HostResourcesCalculator { * Used with exclusive hosts: * Returns the lowest possible real resources we'll get if requesting the given advertised resources */ - NodeResources requestToReal(NodeResources advertisedResources, boolean exclusiveAllocation, boolean bestCase); + NodeResources requestToReal(NodeResources advertisedResources, CloudAccount cloudAccount, + boolean exclusiveAllocation, boolean bestCase); /** * Used with shared hosts: * Returns the advertised resources we need to request to be sure to get at least the given real resources. */ - NodeResources realToRequest(NodeResources realResources, boolean exclusiveAllocation, boolean bestCase); + NodeResources realToRequest(NodeResources realResources, CloudAccount cloudAccount, boolean exclusiveAllocation, boolean bestCase); /** * Returns the disk space to reserve in base2 GB. This space is reserved for use by the host, e.g. for storing diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 8c52f389daf..b149a9af2c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -179,6 +179,13 @@ class NodeAllocation { if (violatesExclusivity(candidate) != NodeCandidate.ExclusivityViolation.NONE) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; if (candidate.violatesSpares) return Retirement.violatesSpares; + + var group = candidate.allocation().get().membership().cluster().group(); + if (cluster.isStateful() && group.isPresent() && requested.count().isPresent()) { + long nodesInGroup = nodes.values().stream().filter(n -> groupOf(n).equals(group) && ! isRetired(n)).count(); + if (nodesInGroup >= requested.groupSize()) + return Retirement.groupSurplus; + } return Retirement.none; } @@ -290,6 +297,10 @@ class NodeAllocation { return candidate.allocation().flatMap(a -> a.membership().cluster().group()); } + private boolean isRetired(NodeCandidate candidate) { + return candidate.allocation().map(a -> a.membership().retired()).orElse(false); + } + private Node resize(Node node) { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requested.resources().get() @@ -463,6 +474,7 @@ class NodeAllocation { violatesHostFlavor("node violates host flavor"), violatesHostFlavorGeneration("node violates host flavor generation"), violatesSpares("node is assigned to a host we want to use as a spare"), + groupSurplus("group has enough nodes"), none(""); private final String description; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index d4c4e86f0a3..d8565b81e41 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -55,6 +55,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat final boolean exclusiveSwitch; /** True if this node belongs to a group which will not be needed after this deployment */ + // TODO: Always false final boolean isSurplus; /** This node does not exist in the node repository yet */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 8e910a4d61c..7ac80dfbdb3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; @@ -187,7 +188,8 @@ public class NodeRepositoryProvisioner implements Provisioner { boolean firstDeployment = nodes.isEmpty(); var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, + nodeRepository, requested.cloudAccount().orElse(CloudAccount.empty)) : new AllocatableResources(nodes, nodeRepository); var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); return within(Limits.of(requested), model, firstDeployment); @@ -199,9 +201,7 @@ public class NodeRepositoryProvisioner implements Provisioner { /** Make the minimal adjustments needed to the current resources to stay within the limits */ - private ClusterResources within(Limits limits, - ClusterModel model, - boolean firstDeployment) { + private ClusterResources within(Limits limits, ClusterModel model, boolean firstDeployment) { if (limits.min().equals(limits.max())) return limits.min(); // Don't change current deployments that are still legal @@ -209,9 +209,7 @@ public class NodeRepositoryProvisioner implements Provisioner { return model.current().advertisedResources(); // Otherwise, find an allocation that preserves the current resources as well as possible - return allocationOptimizer.findBestAllocation(Load.one(), - model, - limits) + return allocationOptimizer.findBestAllocation(Load.one(), model, limits) .orElseThrow(() -> newNoAllocationPossible(model.current().clusterSpec(), limits)) .advertisedResources(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java index 9f7c795cf48..9fbc096667e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.maintenance.CapacityChecker; import java.io.IOException; import java.io.OutputStream; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -46,7 +45,7 @@ public class HostCapacityResponse extends HttpResponse { } private List<Node> parseHostList(String hosts) { - List<String> hostNames = Arrays.asList(hosts.split(",")); + List<String> hostNames = List.of(hosts.split(",")); try { return capacityChecker.nodesFromHostnames(hostNames); } catch (IllegalArgumentException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index c518087f325..cfe4886c903 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.net.URI; -import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; @@ -66,7 +65,7 @@ class NodesResponse extends SlimeJsonResponse { this.wantedDockerTagFlag = PermanentFlags.WANTED_DOCKER_TAG.bindTo(nodeRepository.flagSource()); // Cannot use Set.of() because the nodeRepository account can also be the empty account (at least in tests). - var nonEnclaveAccounts = new HashSet<>(Arrays.asList(CloudAccount.empty, nodeRepository.zone().cloud().account())); + var nonEnclaveAccounts = new HashSet<>(List.of(CloudAccount.empty, nodeRepository.zone().cloud().account())); this.filter = NodesV2ApiHandler.toNodeFilter(request, nonEnclaveAccounts); Cursor root = slime.setObject(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index f653416d973..5795e25d247 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.SystemName; * For running NodeRepository API with some mocked data. * This is used by both NodeAdmin and NodeRepository tests. * - * @author dybis + * @author Haakon Dybdahl */ public class ContainerConfig { @@ -27,6 +27,7 @@ public class ContainerConfig { <accesslog type='disabled'/> <component id='com.yahoo.test.ManualClock'/> <component id='com.yahoo.vespa.curator.mock.MockCurator'/> + <component id='com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb'/> <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockInfraDeployer'/> @@ -34,7 +35,6 @@ public class ContainerConfig { <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/> - <component id='com.yahoo.vespa.hosted.provision.autoscale.QuestMetricsDb'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockMetricsFetcher'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/> <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index 5f0ac4fbcdb..166fcb8edc7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -9,6 +9,7 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -37,7 +38,7 @@ public class MockNameResolver implements NameResolver { Objects.requireNonNull(hostname, "hostname must be non-null"); Arrays.stream(ipAddress).forEach(ip -> Objects.requireNonNull(ip, "ipAddress must be non-null")); records.computeIfAbsent(hostname, (k) -> new HashSet<>()) - .addAll(Arrays.asList(ipAddress)); + .addAll(List.of(ipAddress)); return this; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java index 5409061c441..700a16577af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -38,7 +37,7 @@ public class ServiceMonitorStub implements ServiceMonitor { @Inject @SuppressWarnings("unused") public ServiceMonitorStub(NodeRepository nodeRepository) { - this(Collections.emptyMap(), nodeRepository); + this(Map.of(), nodeRepository); } /** Create a service monitor where all nodes are initially up */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java index 5e4dfdc974d..8318ec65f05 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; @@ -92,7 +93,7 @@ public class ClusterModelTest { return new ClusterModel(nodeRepository, application.with(status), clusterSpec, cluster, - new AllocatableResources(clusterResources(), clusterSpec, nodeRepository), + new AllocatableResources(clusterResources(), clusterSpec, nodeRepository, cluster.cloudAccount().orElse(CloudAccount.empty)), clock, Duration.ofMinutes(10), Duration.ofMinutes(5), timeseries(cluster,100, queryRate, writeRate, clock), ClusterNodesTimeseries.empty()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java index 0fa73aa50a5..f2507786a8b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale.awsnodes; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -43,7 +44,7 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { } @Override - public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive, boolean bestCase) { + public NodeResources requestToReal(NodeResources advertisedResources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { var consideredFlavors = consideredFlavorsGivenAdvertised(advertisedResources); double memoryOverhead = consideredFlavors.stream() .mapToDouble(flavor -> resourcesCalculator.memoryOverhead(flavor, advertisedResources, false)) @@ -56,7 +57,7 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { } @Override - public NodeResources realToRequest(NodeResources realResources, boolean exclusive, boolean bestCase) { + public NodeResources realToRequest(NodeResources realResources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { double chosenMemoryOverhead = bestCase ? Integer.MAX_VALUE : 0; double chosenDiskOverhead = bestCase ? Integer.MAX_VALUE : 0; for (VespaFlavor flavor : consideredFlavorsGivenReal(realResources)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index f25d4cc3c30..72f402ca997 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; @@ -40,6 +41,7 @@ public class ApplicationSerializerTest { new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), IntRange.empty(), true, + Optional.empty(), List.of(), Autoscaling.empty(), ClusterInfo.empty(), @@ -52,6 +54,7 @@ public class ApplicationSerializerTest { new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), IntRange.of(3, 5), false, + Optional.of(CloudAccount.from("aws:123456789012")), List.of(new Autoscaling(Autoscaling.Status.unavailable, "", Optional.of(new ClusterResources(20, 10, @@ -97,6 +100,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); assertEquals(originalCluster.groupSize(), serializedCluster.groupSize()); assertEquals(originalCluster.required(), serializedCluster.required()); + assertEquals(originalCluster.cloudAccount(), serializedCluster.cloudAccount()); assertEquals(originalCluster.suggestions(), serializedCluster.suggestions()); assertEquals(originalCluster.target(), serializedCluster.target()); assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index ff5ffd82bf1..a91902c8eba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -399,7 +399,7 @@ public class DynamicAllocationTest { } @Test - public void node_resources_are_relaxed_in_dev() { + public void node_resources_are_reduced_in_dev() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -419,6 +419,22 @@ public class DynamicAllocationTest { } @Test + public void node_resources_are_reduced_in_staging() { + var resources = new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + tester.makeReadyNodes(3, new Flavor(resources), NodeType.host, 10, true); + tester.activateTenantHosts(); + + ApplicationId application = ProvisioningTester.applicationId(); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("1").build(); + + List<HostSpec> hosts = tester.prepare(application, cluster, 36, 2, resources); + tester.activate(application, hosts); + assertEquals(3, hosts.size()); + assertEquals(1, hosts.stream().map(host -> host.membership().get().cluster().group().get()).distinct().count()); + } + + @Test public void switching_from_legacy_flavor_syntax_to_resources_does_not_cause_reallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index abcef421b4c..78a34326949 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -553,6 +553,34 @@ public class DynamicProvisioningTest { 2, 1, resources); } + @Test + public void split_into_two_groups() { + List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20, 200, 0.1, fast, local))); + + ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false) + .flavors(flavors) + .hostProvisioner(new MockHostProvisioner(flavors)) + .nameResolver(nameResolver) + .build(); + + tester.activateTenantHosts(); + + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); + + System.out.println("Initial deployment ----------------------"); + tester.activate(app1, cluster1, Capacity.from(resources(6, 1, 2, 20, 200, fast, StorageType.any))); + tester.assertNodes("Initial deployment: 1 group", + 6, 1, 2, 20, 200, fast, remote, app1, cluster1); + + System.out.println("Split into 2 groups ---------------------"); + tester.activate(app1, cluster1, Capacity.from(resources(6, 2, 2, 20, 200, fast, StorageType.any))); + tester.assertNodes("Change to 2 groups: Gets 6 active non-retired nodes", + 6, 2, 2, 20, 200, fast, remote, app1, cluster1); + List<Node> retired = tester.nodeRepository().nodes().list().owner(app1).cluster(cluster1.id()).state(Node.State.active).retired().asList(); + assertEquals("... and in addition 3 retired nodes", 3, retired.size()); + } + private ProvisioningTester tester(boolean sharing) { var hostProvisioner = new MockHostProvisioner(new NodeFlavors(ProvisioningTester.createConfig()).getFlavors(), nameResolver, 0); return new ProvisioningTester.Builder().dynamicProvisioning(true, sharing).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 1f8178dff6a..183ff85da47 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -143,6 +144,7 @@ public class DynamicProvisioningTester { cluster.maxResources(), cluster.groupSize(), cluster.required(), + cluster.cloudAccount(), cluster.suggestions(), cluster.target(), cluster.clusterInfo(), @@ -265,12 +267,12 @@ public class DynamicProvisioningTester { } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { + public NodeResources requestToReal(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb()); } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { + public NodeResources realToRequest(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 5f2790e886a..7b690b880c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -288,13 +288,13 @@ public class ProvisioningTest { assertEquals("Superfluous container nodes are also dirtyed", 4-2 + 5-2 + 1 + 4-2, tester.nodeRepository().nodes().list(Node.State.dirty).size()); assertEquals("Superfluous content nodes are retired", - 5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); + 5-3 + 6-3 -1, tester.getNodes(application1, Node.State.active).retired().size()); // increase content slightly SystemState state6 = prepare(application1, 2, 2, 4, 3, defaultResources, tester); tester.activate(application1, state6.allHosts); assertEquals("One content node is unretired", - 5-4 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); + 5-4 + 6-3 -1, tester.getNodes(application1, Node.State.active).retired().size()); // Then reserve more SystemState state7 = prepare(application1, 8, 2, 2, 2, defaultResources, tester); @@ -505,7 +505,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(31, hostFlavor.resources()).activateTenantHosts(); + tester.makeReadyHosts(32, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 48bed11d83f..4ec290dd7ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -811,13 +811,13 @@ public class ProvisioningTester { } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { + public NodeResources requestToReal(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0) ); } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { + public NodeResources realToRequest(NodeResources resources, CloudAccount cloudAccount, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb) .withDiskGb(resources.diskGb() + ( resources.storageType() == local ? localDiskTax : 0) ); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java index c18815fc439..1bf42d72180 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -62,8 +63,8 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 1000, 4)); var calculator = new CompleteResourcesCalculator(hostFlavor); var originalReal = new NodeResources(0.7, 6.0, 12.9, 1.0); - var realToRequest = calculator.realToRequest(originalReal, false, false); - var requestToReal = calculator.requestToReal(realToRequest, false, false); + var realToRequest = calculator.realToRequest(originalReal, CloudAccount.empty, false, false); + var requestToReal = calculator.requestToReal(realToRequest, CloudAccount.empty, false, false); var realResourcesOf = calculator.realResourcesOf(realToRequest); assertEquals(originalReal, requestToReal); assertEquals(originalReal, realResourcesOf); @@ -93,7 +94,8 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { } @Override - public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive, boolean bestCase) { + public NodeResources requestToReal(NodeResources advertisedResources, CloudAccount cloudAccount, + boolean exclusive, boolean bestCase) { double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), advertisedResources, false); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), advertisedResources, false); return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - memoryOverhead) @@ -108,7 +110,8 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { } @Override - public NodeResources realToRequest(NodeResources realResources, boolean exclusive, boolean bestCase) { + public NodeResources realToRequest(NodeResources realResources, CloudAccount cloudAccount, + boolean exclusive, boolean bestCase) { double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), realResources, true); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), realResources, true); return realResources.withMemoryGb(realResources.memoryGb() + memoryOverhead) diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/ApplicationReferenceList.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/ApplicationReferenceList.java index bcbd0d2f3b8..b91271c1adf 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/ApplicationReferenceList.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/ApplicationReferenceList.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Collections; import java.util.List; /** @@ -17,5 +16,5 @@ import java.util.List; @JsonInclude(JsonInclude.Include.NON_NULL) public class ApplicationReferenceList { @JsonProperty("applications") - public List<UrlReference> applicationList = Collections.emptyList(); + public List<UrlReference> applicationList = List.of(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index d2638345f29..bf0de161f73 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -83,8 +82,8 @@ class ClusterApiImpl implements ClusterApi { Collectors.groupingBy( instance -> nodeGroup.contains(instance.hostName()), Collectors.toSet())); - servicesInGroup = serviceInstancesByLocality.getOrDefault(true, Collections.emptySet()); - servicesNotInGroup = serviceInstancesByLocality.getOrDefault(false, Collections.emptySet()); + servicesInGroup = serviceInstancesByLocality.getOrDefault(true, Set.of()); + servicesNotInGroup = serviceInstancesByLocality.getOrDefault(false, Set.of()); int serviceInstances = serviceCluster.serviceInstances().size(); if (clusterParams.size().isPresent() && serviceInstances < clusterParams.size().getAsInt()) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/NodeGroup.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/NodeGroup.java index c61d5cab472..1b1c64c9d9a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/NodeGroup.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/NodeGroup.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -21,7 +20,7 @@ public class NodeGroup { public NodeGroup(ApplicationInstance application, HostName... hostNames) { this.application = application; - this.hostNames.addAll(Arrays.asList(hostNames)); + this.hostNames.addAll(List.of(hostNames)); } public void addNode(HostName hostName) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 354799507ab..fc9715dce5c 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -42,7 +42,6 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -277,7 +276,7 @@ public class OrchestratorImplTest { orchestrator.suspendAll( new HostName("parentHostname"), - Arrays.asList( + List.of( DummyServiceMonitor.TEST1_HOST_NAME, DummyServiceMonitor.TEST3_HOST_NAME, DummyServiceMonitor.TEST6_HOST_NAME)); @@ -317,7 +316,7 @@ public class OrchestratorImplTest { try { orchestrator.suspendAll( new HostName("parentHostname"), - Arrays.asList( + List.of( DummyServiceMonitor.TEST1_HOST_NAME, DummyServiceMonitor.TEST3_HOST_NAME, DummyServiceMonitor.TEST6_HOST_NAME)); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java index c0d6e01104a..8618c6ada90 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java @@ -33,11 +33,11 @@ public class ApplicationApiImplTest { HostName hostName4 = new HostName("host4"); ApplicationInstance applicationInstance = - modelUtils.createApplicationInstance(Arrays.asList( + modelUtils.createApplicationInstance(List.of( modelUtils.createServiceCluster( "cluster-3", new ServiceType("service-type-3"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-1", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-2", hostName2, ServiceStatus.UP) ) @@ -45,7 +45,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-1", new ServiceType("service-type-1"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-3", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-4", hostName3, ServiceStatus.UP) ) @@ -53,7 +53,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-2", new ServiceType("service-type-2"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-5", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-6", hostName2, ServiceStatus.UP) ) @@ -71,7 +71,7 @@ public class ApplicationApiImplTest { try (scopedApi) { // Note: we require the clusters to be in order. List<ClusterApi> clusterApis = scopedApi.applicationApi().getClusters(); - String clusterInfos = clusterApis.stream().map(clusterApi -> clusterApi.clusterInfo()).collect(Collectors.joining(",")); + String clusterInfos = clusterApis.stream().map(ClusterApi::clusterInfo).collect(Collectors.joining(",")); String expectedClusterInfos = Arrays.stream(expectedClusterNumbers) .map(number -> "{ clusterId=cluster-" + number + ", serviceType=service-type-" + number + " }") @@ -92,11 +92,11 @@ public class ApplicationApiImplTest { HostName hostName7 = new HostName("host7"); ApplicationInstance applicationInstance = - modelUtils.createApplicationInstance(Arrays.asList( + modelUtils.createApplicationInstance(List.of( modelUtils.createServiceCluster( "cluster-3", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-30", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-31", hostName2, ServiceStatus.UP) ) @@ -104,7 +104,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-1", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-10", hostName3, ServiceStatus.DOWN), modelUtils.createServiceInstance("config-id-11", hostName4, ServiceStatus.UP) ) @@ -112,7 +112,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-4", new ServiceType("service-type-4"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-40", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-41", hostName2, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-42", hostName3, ServiceStatus.UP), @@ -122,7 +122,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-2", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-20", hostName6, ServiceStatus.DOWN), modelUtils.createServiceInstance("config-id-21", hostName7, ServiceStatus.UP) ) @@ -154,9 +154,9 @@ public class ApplicationApiImplTest { HostName... expectedHostNames) { try (scopedApi) { List<HostName> upStorageNodes = scopedApi.applicationApi().getNoRemarksStorageNodesInGroupInClusterOrder().stream() - .map(storageNode -> storageNode.hostName()) + .map(StorageNode::hostName) .toList(); - assertEquals(Arrays.asList(expectedHostNames), upStorageNodes); + assertEquals(List.of(expectedHostNames), upStorageNodes); } } @@ -191,7 +191,7 @@ public class ApplicationApiImplTest { List<HostName> actualStorageNodes = scopedApi.applicationApi() .getNoRemarksStorageNodesInGroupInClusterOrder() .stream() - .map(storageNode -> storageNode.hostName()) + .map(StorageNode::hostName) .toList(); assertEquals(upStorageNodes, actualStorageNodes); } @@ -204,11 +204,11 @@ public class ApplicationApiImplTest { HostName hostName3 = new HostName("host3"); ApplicationInstance applicationInstance = - modelUtils.createApplicationInstance(Arrays.asList( + modelUtils.createApplicationInstance(List.of( modelUtils.createServiceCluster( "cluster-1", new ServiceType("service-type-1"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-10", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-11", hostName2, ServiceStatus.UP) ) @@ -216,7 +216,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-2", new ServiceType("service-type-2"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-20", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-21", hostName3, ServiceStatus.UP) ) @@ -233,11 +233,11 @@ public class ApplicationApiImplTest { List.of()); verifyNodesInGroupWithoutRemarks( modelUtils.createScopedApplicationApi(applicationInstance, hostName1, hostName2), - Arrays.asList(hostName1, hostName2), + List.of(hostName1, hostName2), List.of()); verifyNodesInGroupWithoutRemarks( modelUtils.createScopedApplicationApi(applicationInstance, hostName1, hostName2, hostName3), - Arrays.asList(hostName1, hostName2), + List.of(hostName1, hostName2), List.of(hostName3)); verifyNodesInGroupWithoutRemarks( modelUtils.createScopedApplicationApi(applicationInstance, hostName3), @@ -268,11 +268,11 @@ public class ApplicationApiImplTest { HostName allowedToBeDownHost7 = new HostName("host7"); ApplicationInstance applicationInstance = - modelUtils.createApplicationInstance(Arrays.asList( + modelUtils.createApplicationInstance(List.of( modelUtils.createServiceCluster( "cluster-4", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-40", allowedToBeDownHost1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-41", noRemarksHost2, ServiceStatus.DOWN) ) @@ -280,7 +280,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-1", new ServiceType("service-type-1"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-10", allowedToBeDownHost1, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-11", allowedToBeDownHost3, ServiceStatus.UP) ) @@ -288,7 +288,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-3", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-30", allowedToBeDownHost4, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-31", noRemarksHost5, ServiceStatus.UP) ) @@ -296,7 +296,7 @@ public class ApplicationApiImplTest { modelUtils.createServiceCluster( "cluster-2", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("config-id-20", noRemarksHost6, ServiceStatus.UP), modelUtils.createServiceInstance("config-id-21", allowedToBeDownHost7, ServiceStatus.UP) ) @@ -343,9 +343,9 @@ public class ApplicationApiImplTest { .applicationApi() .getSuspendedStorageNodesInGroupInReverseClusterOrder() .stream() - .map(storageNode -> storageNode.hostName()) + .map(StorageNode::hostName) .toList(); - assertEquals(Arrays.asList(hostNames), actualStorageNodes); + assertEquals(List.of(hostNames), actualStorageNodes); } } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 7dccaad1b1c..9f8f8a9dd44 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -27,8 +27,6 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Optional; @@ -65,7 +63,7 @@ public class ClusterApiImplTest { ServiceCluster serviceCluster = modelUtils.createServiceCluster( "cluster", new ServiceType("service-type"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("service-1", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("service-2", hostName2, ServiceStatus.DOWN), modelUtils.createServiceInstance("service-3", hostName3, ServiceStatus.UP), @@ -73,7 +71,7 @@ public class ClusterApiImplTest { modelUtils.createServiceInstance("service-5", hostName5, ServiceStatus.UP) ) ); - modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster)); + modelUtils.createApplicationInstance(List.of(serviceCluster)); modelUtils.createNode(hostName1, HostStatus.NO_REMARKS); modelUtils.createNode(hostName2, HostStatus.NO_REMARKS); @@ -269,7 +267,7 @@ public class ClusterApiImplTest { ServiceCluster serviceCluster = modelUtils.createServiceCluster( "cluster", new ServiceType("service-type"), - Arrays.asList( + List.of( modelUtils.createServiceInstance("service-1", hostName1, ServiceStatus.UP), service2, modelUtils.createServiceInstance("service-3", hostName3, ServiceStatus.UP), @@ -278,7 +276,7 @@ public class ClusterApiImplTest { service6 ) ); - modelUtils.createApplicationInstance(Collections.singletonList(serviceCluster)); + modelUtils.createApplicationInstance(List.of(serviceCluster)); modelUtils.createNode(hostName1, HostStatus.NO_REMARKS); modelUtils.createNode(hostName2, HostStatus.NO_REMARKS); @@ -337,7 +335,7 @@ public class ClusterApiImplTest { ServiceCluster serviceCluster = modelUtils.createServiceCluster( "cluster", ServiceType.STORAGE, - Arrays.asList( + List.of( modelUtils.createServiceInstance("storage-1", hostName1, ServiceStatus.UP), modelUtils.createServiceInstance("storage-2", hostName2, ServiceStatus.DOWN) ) diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java index 373d512f811..b2802ad9924 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java @@ -7,8 +7,8 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.TenantId; import org.junit.Test; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import static org.junit.Assert.assertEquals; @@ -27,7 +27,7 @@ public class NodeGroupTest { nodeGroup.addNode(hostName2); // hostnames are sorted (for no good reason other than testability due to stability, readability) - assertEquals(Arrays.asList(hostName1, hostName2, hostName3), nodeGroup.getHostNames()); + assertEquals(List.of(hostName1, hostName2, hostName3), nodeGroup.getHostNames()); assertEquals("host1,host2,host3", nodeGroup.toCommaSeparatedString()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java index 3b1bd7af525..33ec7a5c43f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.TestUtil; import org.junit.Test; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -172,7 +171,7 @@ public class VespaModelUtilTest { @Test public void testGettingClusterControllerInstances() { List<HostName> controllers = VespaModelUtil.getClusterControllerInstancesInOrder(application, CONTENT_CLUSTER_ID); - List<HostName> expectedControllers = Arrays.asList(controller0.hostName(), controller1.hostName()); + List<HostName> expectedControllers = List.of(controller0.hostName(), controller1.hostName()); assertEquals(expectedControllers, controllers); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index 14d5909522a..77fdbe4067d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -20,7 +20,6 @@ import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; -import java.util.Arrays; import java.util.List; import static org.mockito.ArgumentMatchers.any; @@ -56,7 +55,7 @@ public class HostedVespaPolicyTest { ClusterApi clusterApi1 = mock(ClusterApi.class); ClusterApi clusterApi2 = mock(ClusterApi.class); ClusterApi clusterApi3 = mock(ClusterApi.class); - List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + List<ClusterApi> clusterApis = List.of(clusterApi1, clusterApi2, clusterApi3); when(applicationApi.getClusters()).thenReturn(clusterApis); StorageNode storageNode1 = mock(StorageNode.class); @@ -69,11 +68,11 @@ public class HostedVespaPolicyTest { HostName hostName3 = new HostName("storage-3"); when(storageNode1.hostName()).thenReturn(hostName3); - List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3); + List<StorageNode> upStorageNodes = List.of(storageNode1, storageNode3); when(applicationApi.getNoRemarksStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); // setHostState - List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + List<HostName> noRemarksHostNames = List.of(hostName1, hostName2, hostName3); when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames); InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); @@ -108,7 +107,7 @@ public class HostedVespaPolicyTest { ClusterApi clusterApi1 = mock(ClusterApi.class); ClusterApi clusterApi2 = mock(ClusterApi.class); ClusterApi clusterApi3 = mock(ClusterApi.class); - List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + List<ClusterApi> clusterApis = List.of(clusterApi1, clusterApi2, clusterApi3); when(applicationApi.getClusters()).thenReturn(clusterApis); StorageNode storageNode1 = mock(StorageNode.class); @@ -121,10 +120,10 @@ public class HostedVespaPolicyTest { HostName hostName3 = new HostName("storage-3"); when(storageNode1.hostName()).thenReturn(hostName3); - List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3); + List<StorageNode> upStorageNodes = List.of(storageNode1, storageNode3); when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); - List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + List<HostName> noRemarksHostNames = List.of(hostName1, hostName2, hostName3); when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames); InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); @@ -161,7 +160,7 @@ public class HostedVespaPolicyTest { ClusterApi clusterApi1 = mock(ClusterApi.class); ClusterApi clusterApi2 = mock(ClusterApi.class); ClusterApi clusterApi3 = mock(ClusterApi.class); - List<ClusterApi> clusterApis = Arrays.asList(clusterApi1, clusterApi2, clusterApi3); + List<ClusterApi> clusterApis = List.of(clusterApi1, clusterApi2, clusterApi3); when(applicationApi.getClusters()).thenReturn(clusterApis); StorageNode storageNode1 = mock(StorageNode.class); @@ -174,10 +173,10 @@ public class HostedVespaPolicyTest { HostName hostName3 = new HostName("storage-3"); when(storageNode1.hostName()).thenReturn(hostName3); - List<StorageNode> upStorageNodes = Arrays.asList(storageNode1, storageNode3); + List<StorageNode> upStorageNodes = List.of(storageNode1, storageNode3); when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes); - List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3); + List<HostName> noRemarksHostNames = List.of(hostName1, hostName2, hostName3); when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames); InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java index 6a2783949d8..53192b1d40e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java @@ -53,7 +53,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Clock; import java.time.Instant; -import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -278,7 +278,7 @@ class HostRequestHandlerTest { ServiceCluster serviceCluster = new ServiceCluster( new ClusterId("clusterId"), new ServiceType("serviceType"), - Collections.singleton(serviceInstance)); + Set.of(serviceInstance)); serviceInstance.setServiceCluster(serviceCluster); Host host = new Host( @@ -287,7 +287,7 @@ class HostRequestHandlerTest { new ApplicationInstanceReference( new TenantId("tenantId"), new ApplicationInstanceId("applicationId")), - Collections.singletonList(serviceInstance)); + List.of(serviceInstance)); when(orchestrator.getHost(hostName)).thenReturn(host); HttpResponse httpResponse = executeRequest(testDriver, Method.GET, "/orchestrator/v1/hosts/hostname", null); @@ -314,13 +314,14 @@ class HostRequestHandlerTest { assertEquals(409, httpResponse.getStatus()); ByteArrayOutputStream out = new ByteArrayOutputStream(); httpResponse.render(out); - JsonTestHelper.assertJsonEquals("{\n" + - " \"hostname\" : \"hostname\",\n" + - " \"reason\" : {\n" + - " \"constraint\" : \"deadline\",\n" + - " \"message\" : \"resume failed: Timeout Message\"\n" + - " }\n" + - "}", + JsonTestHelper.assertJsonEquals(""" + { + "hostname" : "hostname", + "reason" : { + "constraint" : "deadline", + "message" : "resume failed: Timeout Message" + } + }""", out.toString()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandlerTest.java index 9e5464f3e3a..41bcbb125ee 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceRequestHandlerTest.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.SlobrokApi; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -34,7 +33,7 @@ class InstanceRequestHandlerTest { private static final String APPLICATION_INSTANCE_REFERENCE = "tenant:app:prod:us-west-1:instance"; private static final ApplicationId APPLICATION_ID = ApplicationId.from( "tenant", "app", "instance"); - private static final List<Mirror.Entry> ENTRIES = Arrays.asList( + private static final List<Mirror.Entry> ENTRIES = List.of( new Mirror.Entry("name1", "tcp/spec:1"), new Mirror.Entry("name2", "tcp/spec:2")); private static final ClusterId CLUSTER_ID = new ClusterId("cluster-id"); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java index e0929090a19..d45dc17e84c 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java @@ -34,7 +34,6 @@ import org.mockito.junit.MockitoJUnitRunner; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -393,7 +392,7 @@ public class ZkStatusServiceTest { @SuppressWarnings("varargs") private static <T> List<T> shuffledList(T... values) { //new ArrayList necessary to avoid "write through" behaviour - List<T> list = new ArrayList<>(Arrays.asList(values)); + List<T> list = new ArrayList<>(List.of(values)); Collections.shuffle(list); return list; } diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index deed8d451bd..4cd016215d2 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -4,7 +4,7 @@ namespace storage::spi { -Context::Context(Priority pri, int maxTraceLevel) noexcept +Context::Context(Priority pri, uint32_t maxTraceLevel) noexcept : _priority(pri), _trace(maxTraceLevel), _readConsistency(ReadConsistency::STRONG) diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index 590acd1c549..2b35d5f3a8d 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -38,7 +38,7 @@ using Priority = uint16_t; // 0 - max pri, 255 - min pri // Define this type just because a ton of tests currently use it. struct Trace { - using TraceLevel = int; + using TraceLevel = uint32_t; }; class Context { @@ -48,10 +48,10 @@ class Context { public: Context(Context &&) noexcept = default; Context & operator = (Context &&) noexcept = default; - Context(Priority pri, int maxTraceLevel) noexcept; + Context(Priority pri, uint32_t maxTraceLevel) noexcept; ~Context(); - Priority getPriority() const noexcept { return _priority; } + [[nodiscard]] Priority getPriority() const noexcept { return _priority; } /** * A read operation might choose to relax its consistency requirements, @@ -65,16 +65,16 @@ public: void setReadConsistency(ReadConsistency consistency) noexcept { _readConsistency = consistency; } - ReadConsistency getReadConsistency() const noexcept { + [[nodiscard]] ReadConsistency getReadConsistency() const noexcept { return _readConsistency; } - vespalib::Trace && steal_trace() noexcept { return std::move(_trace); } - vespalib::Trace& getTrace() noexcept { return _trace; } - const vespalib::Trace& getTrace() const noexcept { return _trace; } + [[nodiscard]] vespalib::Trace && steal_trace() noexcept { return std::move(_trace); } + [[nodiscard]] vespalib::Trace& getTrace() noexcept { return _trace; } + [[nodiscard]] const vespalib::Trace& getTrace() const noexcept { return _trace; } - bool shouldTrace(int level) noexcept { return _trace.shouldTrace(level); } - void trace(int level, vespalib::stringref msg, bool addTime = true) { + [[nodiscard]] bool shouldTrace(uint32_t level) const noexcept { return _trace.shouldTrace(level); } + void trace(uint32_t level, vespalib::stringref msg, bool addTime = true) { _trace.trace(level, msg, addTime); } }; diff --git a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Conjunction.java b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Conjunction.java index a5b190d95ab..afe5ea29111 100644 --- a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Conjunction.java +++ b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Conjunction.java @@ -2,7 +2,6 @@ package com.yahoo.document.predicate; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -15,7 +14,7 @@ public class Conjunction extends PredicateOperator { private List<Predicate> operands; public Conjunction(Predicate... operands) { - this(Arrays.asList(operands)); + this(List.of(operands)); } public Conjunction(List<? extends Predicate> operands) { diff --git a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Disjunction.java b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Disjunction.java index 0b03f8afc40..db8392dcdb3 100644 --- a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Disjunction.java +++ b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Disjunction.java @@ -2,7 +2,6 @@ package com.yahoo.document.predicate; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -15,7 +14,7 @@ public class Disjunction extends PredicateOperator { private List<Predicate> operands; public Disjunction(Predicate... operands) { - this(Arrays.asList(operands)); + this(List.of(operands)); } public Disjunction(List<? extends Predicate> operands) { diff --git a/predicate-search-core/src/main/java/com/yahoo/document/predicate/FeatureSet.java b/predicate-search-core/src/main/java/com/yahoo/document/predicate/FeatureSet.java index d2486850f3a..bf1d006fda0 100644 --- a/predicate-search-core/src/main/java/com/yahoo/document/predicate/FeatureSet.java +++ b/predicate-search-core/src/main/java/com/yahoo/document/predicate/FeatureSet.java @@ -1,9 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.predicate; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.TreeSet; @@ -17,7 +17,7 @@ public class FeatureSet extends PredicateValue { private String key; public FeatureSet(String key, String... values) { - this(key, Arrays.asList(values)); + this(key, List.of(values)); } public FeatureSet(String key, Collection<String> values) { @@ -83,10 +83,9 @@ public class FeatureSet extends PredicateValue { if (obj == this) { return true; } - if (!(obj instanceof FeatureSet)) { + if (!(obj instanceof FeatureSet rhs)) { return false; } - FeatureSet rhs = (FeatureSet)obj; if (!key.equals(rhs.key)) { return false; } diff --git a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Negation.java b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Negation.java index 62ad522e655..5e138fe8909 100644 --- a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Negation.java +++ b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Negation.java @@ -28,7 +28,7 @@ public class Negation extends PredicateOperator { @Override public List<Predicate> getOperands() { - return java.util.Arrays.asList(operand); + return java.util.List.of(operand); } @Override diff --git a/predicate-search-core/src/test/java/com/yahoo/document/predicate/ConjunctionTest.java b/predicate-search-core/src/test/java/com/yahoo/document/predicate/ConjunctionTest.java index 56b338453d2..a698248ab6b 100644 --- a/predicate-search-core/src/test/java/com/yahoo/document/predicate/ConjunctionTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/document/predicate/ConjunctionTest.java @@ -3,7 +3,7 @@ package com.yahoo.document.predicate; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -22,18 +22,18 @@ public class ConjunctionTest { Conjunction node = new Conjunction(); Predicate a = SimplePredicates.newString("a"); node.addOperand(a); - assertEquals(Arrays.asList(a), node.getOperands()); + assertEquals(List.of(a), node.getOperands()); Predicate b = SimplePredicates.newString("b"); node.addOperand(b); - assertEquals(Arrays.asList(a, b), node.getOperands()); + assertEquals(List.of(a, b), node.getOperands()); Predicate c = SimplePredicates.newString("c"); Predicate d = SimplePredicates.newString("d"); - node.addOperands(Arrays.asList(c, d)); - assertEquals(Arrays.asList(a, b, c, d), node.getOperands()); + node.addOperands(List.of(c, d)); + assertEquals(List.of(a, b, c, d), node.getOperands()); Predicate e = SimplePredicates.newString("e"); Predicate f = SimplePredicates.newString("f"); - node.setOperands(Arrays.asList(e, f)); - assertEquals(Arrays.asList(e, f), node.getOperands()); + node.setOperands(List.of(e, f)); + assertEquals(List.of(e, f), node.getOperands()); } @Test @@ -41,10 +41,10 @@ public class ConjunctionTest { Predicate foo = SimplePredicates.newString("foo"); Predicate bar = SimplePredicates.newString("bar"); Conjunction node = new Conjunction(foo, bar); - assertEquals(Arrays.asList(foo, bar), node.getOperands()); + assertEquals(List.of(foo, bar), node.getOperands()); - node = new Conjunction(Arrays.asList(foo, bar)); - assertEquals(Arrays.asList(foo, bar), node.getOperands()); + node = new Conjunction(List.of(foo, bar)); + assertEquals(List.of(foo, bar), node.getOperands()); } @Test diff --git a/predicate-search-core/src/test/java/com/yahoo/document/predicate/DisjunctionTest.java b/predicate-search-core/src/test/java/com/yahoo/document/predicate/DisjunctionTest.java index efde8ec7532..fa1cde8fdd3 100644 --- a/predicate-search-core/src/test/java/com/yahoo/document/predicate/DisjunctionTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/document/predicate/DisjunctionTest.java @@ -3,7 +3,7 @@ package com.yahoo.document.predicate; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -22,18 +22,18 @@ public class DisjunctionTest { Disjunction node = new Disjunction(); Predicate a = SimplePredicates.newString("a"); node.addOperand(a); - assertEquals(Arrays.asList(a), node.getOperands()); + assertEquals(List.of(a), node.getOperands()); Predicate b = SimplePredicates.newString("b"); node.addOperand(b); - assertEquals(Arrays.asList(a, b), node.getOperands()); + assertEquals(List.of(a, b), node.getOperands()); Predicate c = SimplePredicates.newString("c"); Predicate d = SimplePredicates.newString("d"); - node.addOperands(Arrays.asList(c, d)); - assertEquals(Arrays.asList(a, b, c, d), node.getOperands()); + node.addOperands(List.of(c, d)); + assertEquals(List.of(a, b, c, d), node.getOperands()); Predicate e = SimplePredicates.newString("e"); Predicate f = SimplePredicates.newString("f"); - node.setOperands(Arrays.asList(e, f)); - assertEquals(Arrays.asList(e, f), node.getOperands()); + node.setOperands(List.of(e, f)); + assertEquals(List.of(e, f), node.getOperands()); } @Test @@ -41,10 +41,10 @@ public class DisjunctionTest { Predicate foo = SimplePredicates.newString("foo"); Predicate bar = SimplePredicates.newString("bar"); Disjunction node = new Disjunction(foo, bar); - assertEquals(Arrays.asList(foo, bar), node.getOperands()); + assertEquals(List.of(foo, bar), node.getOperands()); - node = new Disjunction(Arrays.asList(foo, bar)); - assertEquals(Arrays.asList(foo, bar), node.getOperands()); + node = new Disjunction(List.of(foo, bar)); + assertEquals(List.of(foo, bar), node.getOperands()); } @Test diff --git a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureConjunctionTest.java b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureConjunctionTest.java index 38b8d668a18..35dab62925f 100644 --- a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureConjunctionTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureConjunctionTest.java @@ -3,8 +3,7 @@ package com.yahoo.document.predicate; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import static com.yahoo.document.predicate.Predicates.feature; import static com.yahoo.document.predicate.Predicates.not; @@ -17,7 +16,7 @@ public class FeatureConjunctionTest { @Test void require_that_featureconjunction_with_valid_operands_can_be_constructed() { - new FeatureConjunction(Arrays.asList( + new FeatureConjunction(List.of( not(feature("a").inSet("1")), feature("b").inSet("1"))); } @@ -25,7 +24,7 @@ public class FeatureConjunctionTest { @Test void require_that_constructor_throws_exception_if_all_operands_are_not_featuresets() { assertThrows(IllegalArgumentException.class, () -> { - new FeatureConjunction(Arrays.asList( + new FeatureConjunction(List.of( not(feature("a").inSet("1")), feature("b").inRange(1, 2))); }); @@ -34,28 +33,28 @@ public class FeatureConjunctionTest { @Test void require_that_constructor_throws_exception_if_single_operand() { assertThrows(IllegalArgumentException.class, () -> { - new FeatureConjunction(Arrays.asList(feature("a").inSet("1"))); + new FeatureConjunction(List.of(feature("a").inSet("1"))); }); } @Test void require_that_constructor_throws_exception_if_no_operands() { assertThrows(IllegalArgumentException.class, () -> { - new FeatureConjunction(Collections.emptyList()); + new FeatureConjunction(List.of()); }); } @Test void require_that_contructor_throws_exception_if_featuresets_contain_multiple_values() { assertThrows(IllegalArgumentException.class, () -> { - new FeatureConjunction(Arrays.asList(feature("a").inSet("1"), feature("b").inSet("2", "3"))); + new FeatureConjunction(List.of(feature("a").inSet("1"), feature("b").inSet("2", "3"))); }); } @Test void require_that_constructor_throws_exception_if_featureset_keys_are_not_unique() { assertThrows(IllegalArgumentException.class, () -> { - new FeatureConjunction(Arrays.asList( + new FeatureConjunction(List.of( not(feature("a").inSet("1")), feature("a").inSet("2"))); }); diff --git a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureRangeTest.java b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureRangeTest.java index 06c1a445494..3977ad02999 100644 --- a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureRangeTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureRangeTest.java @@ -3,7 +3,7 @@ package com.yahoo.document.predicate; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -249,15 +249,15 @@ public class FeatureRangeTest { @Test void requireThatFeatureRangeCanBeBuiltFromMixedInNode() { assertEquals(new FeatureRange("foo", 10L, 19L), - FeatureRange.buildFromMixedIn("foo", Arrays.asList("foo=10-19"), 10)); + FeatureRange.buildFromMixedIn("foo", List.of("foo=10-19"), 10)); assertEquals(new FeatureRange("foo", -19L, -10L), - FeatureRange.buildFromMixedIn("foo", Arrays.asList("foo=-10-19"), 10)); + FeatureRange.buildFromMixedIn("foo", List.of("foo=-10-19"), 10)); assertEquals(new FeatureRange("foo", 10L, 19L), - FeatureRange.buildFromMixedIn("foo", Arrays.asList("foo=10,10,9"), 10)); + FeatureRange.buildFromMixedIn("foo", List.of("foo=10,10,9"), 10)); assertEquals(new FeatureRange("foo", 10L, 19L), - FeatureRange.buildFromMixedIn("foo", Arrays.asList("foo=10,10,1073741833"), 10)); + FeatureRange.buildFromMixedIn("foo", List.of("foo=10,10,1073741833"), 10)); assertEquals(new FeatureRange("foo", 10L, 19L), - FeatureRange.buildFromMixedIn("foo", Arrays.asList("foo=10,10,2147483648"), 10)); + FeatureRange.buildFromMixedIn("foo", List.of("foo=10,10,2147483648"), 10)); } } diff --git a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureSetTest.java b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureSetTest.java index b61f843c10e..31cfe4318fb 100644 --- a/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureSetTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/document/predicate/FeatureSetTest.java @@ -4,7 +4,6 @@ package com.yahoo.document.predicate; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -25,35 +24,35 @@ public class FeatureSetTest { void requireThatAccessorsWork() { FeatureSet node = new FeatureSet("key", "valueA", "valueB"); assertEquals("key", node.getKey()); - assertValues(Arrays.asList("valueA", "valueB"), node); + assertValues(List.of("valueA", "valueB"), node); node.addValue("valueC"); - assertValues(Arrays.asList("valueA", "valueB", "valueC"), node); - node.addValues(Arrays.asList("valueD", "valueE")); - assertValues(Arrays.asList("valueA", "valueB", "valueC", "valueD", "valueE"), node); - node.setValues(Arrays.asList("valueF", "valueG")); - assertValues(Arrays.asList("valueF", "valueG"), node); + assertValues(List.of("valueA", "valueB", "valueC"), node); + node.addValues(List.of("valueD", "valueE")); + assertValues(List.of("valueA", "valueB", "valueC", "valueD", "valueE"), node); + node.setValues(List.of("valueF", "valueG")); + assertValues(List.of("valueF", "valueG"), node); } @Test void requireThatValueSetIsMutable() { FeatureSet node = new FeatureSet("key"); node.getValues().add("valueA"); - assertValues(Arrays.asList("valueA"), node); + assertValues(List.of("valueA"), node); node = new FeatureSet("key", "valueA"); node.getValues().add("valueB"); - assertValues(Arrays.asList("valueA", "valueB"), node); + assertValues(List.of("valueA", "valueB"), node); } @Test void requireThatConstructorsWork() { FeatureSet node = new FeatureSet("key", "valueA", "valueB"); assertEquals("key", node.getKey()); - assertValues(Arrays.asList("valueA", "valueB"), node); + assertValues(List.of("valueA", "valueB"), node); - node = new FeatureSet("key", Arrays.asList("valueA", "valueB")); + node = new FeatureSet("key", List.of("valueA", "valueB")); assertEquals("key", node.getKey()); - assertValues(Arrays.asList("valueA", "valueB"), node); + assertValues(List.of("valueA", "valueB"), node); } @Test @@ -123,7 +122,7 @@ public class FeatureSetTest { } catch (NullPointerException e) { assertEquals("value", e.getMessage()); } - assertValues(Arrays.asList("bar"), node); + assertValues(List.of("bar"), node); } @Test diff --git a/predicate-search-core/src/test/java/com/yahoo/search/predicate/PredicateQueryParserTest.java b/predicate-search-core/src/test/java/com/yahoo/search/predicate/PredicateQueryParserTest.java index 8bc434218b7..b8ec623410c 100644 --- a/predicate-search-core/src/test/java/com/yahoo/search/predicate/PredicateQueryParserTest.java +++ b/predicate-search-core/src/test/java/com/yahoo/search/predicate/PredicateQueryParserTest.java @@ -4,7 +4,6 @@ package com.yahoo.search.predicate; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,7 +33,7 @@ public class PredicateQueryParserTest { (k, v, s) -> result.add(String.format("%s:%s:%#x", k, v, s)), (k, v, s) -> result.add(String.format("%s:%d:%#x", k, v, s))); - assertEquals(result, Arrays.asList( + assertEquals(result, List.of( "k1:value1:0x1", "k2:value2:0x3", "range1:123456789123:0xffff", "range2:0:0xffffffffffffffff")); } diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnalyzerTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnalyzerTest.java index 401c006b273..bc330a7d1f3 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnalyzerTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnalyzerTest.java @@ -6,7 +6,8 @@ import com.yahoo.document.predicate.Predicate; import com.yahoo.document.predicate.PredicateOperator; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.document.predicate.Predicates.and; import static com.yahoo.document.predicate.Predicates.feature; @@ -208,7 +209,7 @@ public class PredicateTreeAnalyzerTest { } private static FeatureConjunction conj(Predicate... operands) { - return new FeatureConjunction(Arrays.asList(operands)); + return new FeatureConjunction(List.of(operands)); } private static void assertSizeMapContains(PredicateTreeAnalyzerResult r, PredicateSelector selector, int expectedValue) { @@ -217,18 +218,13 @@ public class PredicateTreeAnalyzerTest { assertEquals(expectedValue, actualValue.intValue()); } - private static class PredicateSelector { - public final Predicate predicate; - - public PredicateSelector(Predicate predicate) { - this.predicate = predicate; - } + private record PredicateSelector(Predicate predicate) { public PredicateSelector child(int index) { - PredicateOperator op = (PredicateOperator) predicate; - return new PredicateSelector(op.getOperands().get(index)); + PredicateOperator op = (PredicateOperator) predicate; + return new PredicateSelector(op.getOperands().get(index)); + } } - } private static PredicateSelector pred(Predicate p) { return new PredicateSelector(p); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java index 18cdc4defff..237aeda4fb3 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/annotator/PredicateTreeAnnotatorTest.java @@ -13,7 +13,6 @@ import com.yahoo.search.predicate.index.IntervalWithBounds; import com.yahoo.search.predicate.index.conjunction.IndexableFeatureConjunction; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -229,7 +228,7 @@ public class PredicateTreeAnnotatorTest { long hash = PredicateHash.hash64(feature); List<Integer> actualIntervals = r.intervalMap.get(hash); assertNotNull(actualIntervals); - assertArrayEquals(Ints.toArray(Arrays.asList(expectedIntervals)), Ints.toArray(actualIntervals)); + assertArrayEquals(Ints.toArray(List.of(expectedIntervals)), Ints.toArray(actualIntervals)); } private static void assertBoundsContains(PredicateTreeAnnotations r, String feature, IntervalWithBounds expectedBounds) { @@ -258,11 +257,11 @@ public class PredicateTreeAnnotatorTest { private static FeatureRange range(String key, Long lower, Long upper, RangePartition... partitions) { FeatureRange range = new FeatureRange(key, lower, upper); - Arrays.asList(partitions).forEach(range::addPartition); + List.of(partitions).forEach(range::addPartition); return range; } private static FeatureConjunction conj(Predicate... operands) { - return new FeatureConjunction(Arrays.asList(operands)); + return new FeatureConjunction(List.of(operands)); } } diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java index 68f692e3119..cf7e286f89f 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/CachedPostingListCounterTest.java @@ -6,7 +6,6 @@ import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -107,12 +106,12 @@ public class CachedPostingListCounterTest { } private static List<PostingList> list(PostingList... postingLists) { - return Arrays.asList(postingLists); + return List.of(postingLists); } private static PostingList postingList(Integer... docIds) { PostingList postingList = mock(PostingList.class); - when(postingList.getDocIds()).thenReturn(Ints.toArray(Arrays.asList((docIds)))); + when(postingList.getDocIds()).thenReturn(Ints.toArray(List.of((docIds)))); return postingList; } diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java index 8b2a5cc1d7e..26f191e13b6 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java @@ -4,7 +4,8 @@ package com.yahoo.search.predicate.index; import com.yahoo.search.predicate.SubqueryBitmap; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -13,9 +14,9 @@ public class IntervalPostingListTest { @Test void requireThatPostingListCanIterate() { PredicateIntervalStore.Builder builder = new PredicateIntervalStore.Builder(); - int ref1 = builder.insert(Arrays.asList(0x1ffff)); - int ref2 = builder.insert(Arrays.asList(0x1ffff)); - int ref3 = builder.insert(Arrays.asList(0x10001, 0x2ffff)); + int ref1 = builder.insert(List.of(0x1ffff)); + int ref2 = builder.insert(List.of(0x1ffff)); + int ref3 = builder.insert(List.of(0x10001, 0x2ffff)); IntervalPostingList postingList = new IntervalPostingList( builder.build(), new int[]{2, 4, 6}, new int[]{ref1, ref2, ref3}, SubqueryBitmap.ALL_SUBQUERIES); assertEquals(-1, postingList.getDocId()); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateIntervalStoreTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateIntervalStoreTest.java index bfc4635aa51..c02c18ae1d7 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateIntervalStoreTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateIntervalStoreTest.java @@ -6,7 +6,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static com.yahoo.search.predicate.serialization.SerializationTestHelper.assertSerializationDeserializationMatches; @@ -49,9 +48,9 @@ public class PredicateIntervalStoreTest { @Test void requireThatSerializationAndDeserializationRetainIntervals() throws IOException { PredicateIntervalStore.Builder builder = new PredicateIntervalStore.Builder(); - builder.insert(Arrays.asList(0x00010001, 0x00020002)); - builder.insert(Arrays.asList(0x00010001, 0x00020002, 0x00030003)); - builder.insert(Arrays.asList(0x0fffffff, 0x00020002, 0x00030003)); + builder.insert(List.of(0x00010001, 0x00020002)); + builder.insert(List.of(0x00010001, 0x00020002, 0x00030003)); + builder.insert(List.of(0x0fffffff, 0x00020002, 0x00030003)); PredicateIntervalStore store = builder.build(); assertSerializationDeserializationMatches( store, PredicateIntervalStore::writeToOutputStream, PredicateIntervalStore::fromInputStream); @@ -60,8 +59,8 @@ public class PredicateIntervalStoreTest { @Test void requireThatEqualIntervalListsReturnsSameReference() { PredicateIntervalStore.Builder builder = new PredicateIntervalStore.Builder(); - List<Integer> intervals1 = Arrays.asList(0x00010001, 0x00020002); - List<Integer> intervals2 = Arrays.asList(0x00010001, 0x00020002); + List<Integer> intervals1 = List.of(0x00010001, 0x00020002); + List<Integer> intervals2 = List.of(0x00010001, 0x00020002); int ref1 = builder.insert(intervals1); int ref2 = builder.insert(intervals2); PredicateIntervalStore store = builder.build(); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateRangeTermExpanderTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateRangeTermExpanderTest.java index 3c0a2290817..0b8076b310c 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateRangeTermExpanderTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateRangeTermExpanderTest.java @@ -4,8 +4,8 @@ package com.yahoo.search.predicate.index; import com.yahoo.document.predicate.PredicateHash; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Iterator; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -16,7 +16,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSmallRangeIsExpanded() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=40-49", "key=0-99", "key=0-999", @@ -46,7 +46,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatLargeRangeIsExpanded() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=123456789012345670-123456789012345679", "key=123456789012345600-123456789012345699", "key=123456789012345000-123456789012345999", @@ -86,7 +86,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSmallNegativeRangeIsExpanded() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=-49-40", "key=-99-0", "key=-999-0", @@ -126,7 +126,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatMinRangeMinus9IsExpanded() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=-9223372036854775799-9223372036854775790", "key=-9223372036854775799-9223372036854775700").iterator(); expander.expand("key", -9223372036854775799L, range -> assertEquals(PredicateHash.hash64(expectedLabels.next()), range), @@ -150,7 +150,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSmallRangeIsExpandedInArity2() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(2); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=42-43", "key=40-43", "key=40-47", @@ -225,7 +225,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSmallNegativeRangeIsExpandedInArity2() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(2); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=-43-42", "key=-43-40", "key=-47-40", @@ -300,7 +300,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatUpperBoundIsUsed() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10, -99, 9999); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=40-49", "key=0-99", "key=0-999", @@ -316,7 +316,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatLowerBoundIsUsed() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10, -9999, 99); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=-49-40", "key=-99-0", "key=-999-0", @@ -339,7 +339,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatUpperAndLowerBoundGreaterThan0Works() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10, 100, 9999); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=140-149", "key=100-199", "key=0-999", @@ -355,7 +355,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSearchCloseToUnevenUpperBoundIsSensible() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10, -99, 1234); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=40-49", "key=0-99", "key=0-999", @@ -371,7 +371,7 @@ public class PredicateRangeTermExpanderTest { @Test void requireThatSearchCloseToMaxUnevenUpperBoundIsSensible() { PredicateRangeTermExpander expander = new PredicateRangeTermExpander(10, 0, 9223372036854771234L); - Iterator<String> expectedLabels = Arrays.asList( + Iterator<String> expectedLabels = List.of( "key=9223372036854770000-9223372036854770009", "key=9223372036854770000-9223372036854770099", "key=9223372036854770000-9223372036854770999").iterator(); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateSearchTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateSearchTest.java index 62dde7ac585..d520980193e 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateSearchTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/PredicateSearchTest.java @@ -32,7 +32,7 @@ public class PredicateSearchTest { entry(0, 0x000100ff), entry(1, 0x00010001, 0x000200ff), entry(2, 0x00010042))); - assertEquals(Arrays.asList(new Hit(0), new Hit(1)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0), new Hit(1)).toString(), search.stream().toList().toString()); } @Test @@ -43,7 +43,7 @@ public class PredicateSearchTest { SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x000100ff), entry(1, 0x000100ff))); - assertEquals(Arrays.asList(new Hit(1)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(1)).toString(), search.stream().toList().toString()); } @Test @@ -54,7 +54,7 @@ public class PredicateSearchTest { entry(0, 0x00010001)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x000200ff))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } @Test @@ -67,7 +67,7 @@ public class PredicateSearchTest { postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(1, 0x000100ff), entry(2, 0x000100ff))); - assertEquals(Arrays.asList(new Hit(1), new Hit(2), new Hit(3)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(1), new Hit(2), new Hit(3)).toString(), search.stream().toList().toString()); } @Test @@ -75,7 +75,7 @@ public class PredicateSearchTest { PredicateSearch search = createPredicateSearch( new byte[0], postingList(SubqueryBitmap.ALL_SUBQUERIES)); - assertEquals(Arrays.asList().toString(), search.stream().toList().toString()); + assertEquals(List.of().toString(), search.stream().toList().toString()); } @Test @@ -87,7 +87,7 @@ public class PredicateSearchTest { entry(1, 0x000100ff)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(2, 0x000100ff))); - assertEquals(Arrays.asList(new Hit(0), new Hit(1), new Hit(2)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0), new Hit(1), new Hit(2)).toString(), search.stream().toList().toString()); } @Test @@ -110,7 +110,7 @@ public class PredicateSearchTest { entry(1, 0x000100ff), entry(3, 0x000100ff))); assertEquals( - Arrays.asList(new Hit(0), new Hit(1), new Hit(2), new Hit(3)).toString(), + List.of(new Hit(0), new Hit(1), new Hit(2), new Hit(3)).toString(), search.stream().toList().toString()); } @@ -121,7 +121,7 @@ public class PredicateSearchTest { postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00010001), entry(1, 0x000200ff))); - assertEquals(Arrays.asList().toString(), search.stream().toList().toString()); + assertEquals(List.of().toString(), search.stream().toList().toString()); } @Test @@ -134,7 +134,7 @@ public class PredicateSearchTest { entry(0, 0x000300ff)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00020002))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } @Test @@ -143,7 +143,7 @@ public class PredicateSearchTest { new byte[]{1}, postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00010001, 0x00020002, 0x00030003, 0x000100ff, 0x00040004, 0x00050005, 0x00060006))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } @Test @@ -154,7 +154,7 @@ public class PredicateSearchTest { entry(0, 0x00010001)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00010000, 0x00ff0001))); - assertEquals(Arrays.asList().toString(), search.stream().toList().toString()); + assertEquals(List.of().toString(), search.stream().toList().toString()); } @Test @@ -163,7 +163,7 @@ public class PredicateSearchTest { new byte[]{1}, postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00010000, 0x00ff0001))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } @Test @@ -172,7 +172,7 @@ public class PredicateSearchTest { new byte[]{1}, postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00020001, 0x00ff0001))); - assertEquals(Arrays.asList().toString(), search.stream().toList().toString()); + assertEquals(List.of().toString(), search.stream().toList().toString()); } @Test @@ -183,7 +183,7 @@ public class PredicateSearchTest { entry(0, 0x00010001)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00fe0001, 0x00ff00fe))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } @Test @@ -202,14 +202,14 @@ public class PredicateSearchTest { entry(0, 0x00010008, 0x00060006)), postingList(SubqueryBitmap.ALL_SUBQUERIES, entry(0, 0x00020002, 0x000700ff))); - assertEquals(Arrays.asList(new Hit(0)).toString(), search.stream().toList().toString()); + assertEquals(List.of(new Hit(0)).toString(), search.stream().toList().toString()); } private static PredicateSearch createPredicateSearch(byte[] minFeatures, PostingList... postingLists) { byte[] nPostingListsForDocument = new byte[minFeatures.length]; short[] intervalEnds = new short[minFeatures.length]; Arrays.fill(intervalEnds, (short) 0xFF); - List<PostingList> list = Arrays.asList(postingLists); + List<PostingList> list = List.of(postingLists); for (PostingList postingList : postingLists) { for (int id : postingList.getDocIds()) { nPostingListsForDocument[id]++; diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/ZstarCompressedPostingListTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/ZstarCompressedPostingListTest.java index 408d4f43489..43fd35d8689 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/ZstarCompressedPostingListTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/ZstarCompressedPostingListTest.java @@ -3,7 +3,8 @@ package com.yahoo.search.predicate.index; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -14,9 +15,9 @@ public class ZstarCompressedPostingListTest { @Test void requireThatPostingListCanIterate() { PredicateIntervalStore.Builder builder = new PredicateIntervalStore.Builder(); - int ref1 = builder.insert(Arrays.asList(0x10000)); - int ref2 = builder.insert(Arrays.asList(0x10000, 0x0ffff)); - int ref3 = builder.insert(Arrays.asList(0x10000, 0x00003, 0x40003, 0x60005)); + int ref1 = builder.insert(List.of(0x10000)); + int ref2 = builder.insert(List.of(0x10000, 0x0ffff)); + int ref3 = builder.insert(List.of(0x10000, 0x00003, 0x40003, 0x60005)); ZstarCompressedPostingList postingList = new ZstarCompressedPostingList( builder.build(), new int[]{2, 4, 6}, new int[]{ref1, ref2, ref3}); assertEquals(-1, postingList.getDocId()); diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexTest.java index 33265d9bb61..c0334b81c59 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/conjunction/ConjunctionIndexTest.java @@ -343,7 +343,7 @@ public class ConjunctionIndexTest { } private static List<ConjunctionHit> matchingConjunctionList(ConjunctionHit... conjunctionHits) { - return Arrays.asList(conjunctionHits); + return List.of(conjunctionHits); } private static void assertHitsEquals(List<ConjunctionHit> hits, IndexableFeatureConjunction... conjunctions) { @@ -356,15 +356,11 @@ public class ConjunctionIndexTest { } private static void assertHitsEquals(List<ConjunctionHit> expectedHits, List<ConjunctionHit> hits) { - Collections.sort(expectedHits); - Collections.sort(hits); - assertArrayEquals( - expectedHits.toArray(new ConjunctionHit[expectedHits.size()]), - hits.toArray(new ConjunctionHit[expectedHits.size()])); + assertEquals(expectedHits.stream().sorted().toList(), hits.stream().sorted().toList()); } private static FeatureConjunction conj(Predicate... operands) { - return new FeatureConjunction(Arrays.asList(operands)); + return new FeatureConjunction(List.of(operands)); } private static IndexableFeatureConjunction indexableConj(FeatureConjunction conjunction) { diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/optimization/FeatureConjunctionTransformerTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/optimization/FeatureConjunctionTransformerTest.java index 2e9c9a32410..c8c4b1f1911 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/optimization/FeatureConjunctionTransformerTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/optimization/FeatureConjunctionTransformerTest.java @@ -7,7 +7,8 @@ import com.yahoo.document.predicate.FeatureSet; import com.yahoo.document.predicate.Predicate; import org.junit.jupiter.api.Test; -import java.util.Arrays; + +import java.util.List; import static com.yahoo.document.predicate.Predicates.and; import static com.yahoo.document.predicate.Predicates.not; @@ -101,7 +102,7 @@ public class FeatureConjunctionTransformerTest { } private static FeatureConjunction conj(Predicate... operands) { - return new FeatureConjunction(Arrays.asList(operands)); + return new FeatureConjunction(List.of(operands)); } private static FeatureSet featureSet(int id, String... values) { diff --git a/screwdriver.yaml b/screwdriver.yaml index 32ad8288d70..4edef83b63b 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -111,6 +111,8 @@ jobs: - get-vespa-version: | if [[ -z "$SD_PULL_REQUEST" ]]; then VERSION_INFO=$(screwdriver/factory-command.sh create-build) + echo "Version information from Factory : $VERSION_INFO" + VESPA_VERSION=$(jq -re '.version' <<< "$VERSION_INFO") VESPA_REF=$(jq -re '.commits|.[]|select(.repo=="vespa")|.ref' <<< "$VERSION_INFO") SYSTEM_TEST_REF=$(jq -re '.commits|.[]|select(.repo=="system-test")|.ref' <<< "$VERSION_INFO") diff --git a/screwdriver/factory-command.sh b/screwdriver/factory-command.sh index 3b739b9f5c9..a09900ec0a0 100755 --- a/screwdriver/factory-command.sh +++ b/screwdriver/factory-command.sh @@ -12,7 +12,24 @@ FACTORY_API="https://factory.vespa.aws-us-east-1a.vespa.oath.cloud/api/factory/v COOKIEJAR=$(pwd)/jar.txt trap "rm -f $COOKIEJAR" EXIT -SESSION_TOKEN=$(curl -s -H 'Content-Type: application/json' -H 'Accept: application/json' -d "{ \"username\": \"svc-okta-vespa-factory\", \"password\": \"$SVC_OKTA_VESPA_FACTORY_TOKEN\" }" https://ouryahoo.okta.com/api/v1/authn | jq -re '.sessionToken') +SESSION_TOKEN=null +WAIT_UNTIL=$(( $(date +%s) + 120 )) +set +e +while [[ $SESSION_TOKEN == null ]]; do + SESSION_TOKEN=$(curl -s -H 'Content-Type: application/json' -H 'Accept: application/json' -d "{ \"username\": \"svc-okta-vespa-factory\", \"password\": \"$SVC_OKTA_VESPA_FACTORY_TOKEN\" }" https://ouryahoo.okta.com/api/v1/authn | jq -re '.sessionToken') + + if [[ $SESSION_TOKEN == null ]]; then + if [[ $(date +%s) -ge $WAIT_UNTIL ]]; then + echo "Could not fetch session token from Okta: SESSION_TOKEN=$SESSION_TOKEN" + exit 1 + else + echo "Invalid SESSION_TOKEN=$SESSION_TOKEN . Trying again ..." >&2 + sleep 3 + fi + fi +done +set -e + LOCATION=$(curl -s -i -c $COOKIEJAR "https://factory.vespa.aws-us-east-1a.vespa.oath.cloud/login" | grep location | awk '{print $2}' | tr -d '\r') curl -sL -b $COOKIEJAR -c $COOKIEJAR "$LOCATION&sessionToken=$SESSION_TOKEN" &> /dev/null diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt index a26a9e463d6..40bdcbaf1b1 100644 --- a/searchcore/src/apps/proton/CMakeLists.txt +++ b/searchcore/src/apps/proton/CMakeLists.txt @@ -23,4 +23,5 @@ vespa_add_executable(searchcore_proton_app searchcore_grouping searchcore_proton_metrics storageserver_storageapp + absl::failure_signal_handler ) diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index e967c012bbe..de256ebf0d9 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -12,6 +12,7 @@ #include <vespa/config/common/configcontext.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/file.h> +#include <absl/debugging/failure_signal_handler.h> #include <filesystem> #include <iostream> #include <thread> @@ -53,6 +54,20 @@ public: void App::setupSignals() { + absl::FailureSignalHandlerOptions opts; + // Sanitizers set up their own signal handler, so we must ensure that the failure signal + // handler calls this when it's done, or we won't get a proper report. + opts.call_previous_handler = true; + // Ideally we'd use an alternate stack to have well-defined reporting when a + // thread runs out of stack space (infinite recursion bug etc.), but for some + // reason this seems to negatively affect stack walking and give very incomplete + // traces. So until this is resolved, use the thread's own stack. + opts.use_alternate_stack = false; + absl::InstallFailureSignalHandler(opts); + + // Install our own signal handlers _after_ the failure handler, as the sentinel uses + // SIGTERM as a "friendly poke for shutdown" signal and the Abseil failure handler + // always dumps stack when intercepting this signal (since it's considered fatal). SIG::PIPE.ignore(); SIG::INT.hook(); SIG::TERM.hook(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 87004d7e5f2..0c986422be6 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -2,10 +2,11 @@ #include "lid_allocator.h" #include <vespa/searchlib/common/bitvectoriterator.h> -#include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/fef/matchdata.h> -#include <vespa/searchlib/queryeval/full_search.h> +#include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> +#include <vespa/searchlib/queryeval/full_search.h> #include <mutex> #include <vespa/log/log.h> @@ -19,6 +20,8 @@ using search::queryeval::SearchIterator; using search::queryeval::SimpleLeafBlueprint; using vespalib::GenerationHolder; +using namespace search::queryeval::flow; + namespace proton::documentmetastore { LidAllocator::LidAllocator(uint32_t size, @@ -206,7 +209,8 @@ private: return search::BitVectorIterator::create(&_activeLids, get_docid_limit(), *tfmd, strict); } FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - return default_flow_stats(docid_limit, _activeLids.size(), 0); + double rel_est = abs_to_rel_est(_activeLids.size(), docid_limit); + return {rel_est, bitvector_cost(), bitvector_strict_cost(rel_est)}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda) const override diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index c03d93b6480..0b2660824c0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -88,6 +88,7 @@ private: double w = getWeightFromNode(*node).percent(); eq->addTerm(build(_requestContext, *node, _context), w / eqw); } + _result->setDocIdLimit(_context.getDocIdLimit()); n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit()); } @@ -123,6 +124,7 @@ private: indexBlueprint = _context.getIndexes().createBlueprint(_requestContext, indexFields, n); } _result = mixer.mix(std::move(indexBlueprint)); + _result->setDocIdLimit(_context.getDocIdLimit()); n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit()); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp index 60c2e869e79..47a9f3dd43d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp @@ -93,7 +93,8 @@ void ProtonTermData::setDocumentFrequency(uint32_t estHits, uint32_t docIdLimit) { if (docIdLimit > 1) { - propagate_document_frequency(estHits, docIdLimit - 1); + uint32_t total_doc_count = docIdLimit - 1; + propagate_document_frequency(std::min(estHits, total_doc_count), total_doc_count); } else { propagate_document_frequency(0, 1); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 093e65b2e4d..840eacd9dd9 100755..100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -15,7 +15,6 @@ import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpre import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.List; @@ -50,7 +49,7 @@ public class ExpressionFunction { * @param body the ranking expression that defines this function */ public ExpressionFunction(String name, RankingExpression body) { - this(name, Collections.emptyList(), body); + this(name, List.of(), body); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java index 1186541b9c0..c7231ecd800 100755..100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java @@ -7,7 +7,6 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Objects; @@ -36,7 +35,7 @@ public final class EmbracedNode extends CompositeNode { @Override public List<ExpressionNode> children() { - return Collections.singletonList(value); + return List.of(value); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java index e7db8848be5..e93edb71ac6 100755..100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java @@ -11,7 +11,6 @@ import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.tensor.functions.Join; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Objects; @@ -34,7 +33,7 @@ public final class FunctionNode extends CompositeNode { public FunctionNode(Function function, ExpressionNode argument) { if (function.arity() != 1) throw new IllegalArgumentException(function + " is not unary"); this.function = function; - this.arguments = new Arguments(Collections.singletonList(argument)); + this.arguments = new Arguments(List.of(argument)); } /** Creates a binary function node */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index a2cf662a255..be0a532128f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -4,8 +4,8 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -23,12 +23,12 @@ public class FunctionReferenceContext { /** Create a context for a single serialization task */ public FunctionReferenceContext() { - this(Collections.emptyList()); + this(List.of()); } /** Create a context for a single serialization task */ public FunctionReferenceContext(Collection<ExpressionFunction> functions) { - this(toMap(functions), Collections.emptyMap()); + this(toMap(functions), Map.of()); } public FunctionReferenceContext(Collection<ExpressionFunction> functions, Map<String, String> bindings) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java index bef19a656f8..49311b1553b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java @@ -8,7 +8,6 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Objects; @@ -39,7 +38,7 @@ public class GeneratorLambdaFunctionNode extends CompositeNode { @Override public List<ExpressionNode> children() { - return Collections.singletonList(generator); + return List.of(generator); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index 0f1331515cc..b84bb655140 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -9,7 +9,6 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.tensor.functions.Generate; -import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.List; @@ -50,7 +49,7 @@ public class LambdaFunctionNode extends CompositeNode { @Override public List<ExpressionNode> children() { - return Collections.singletonList(functionExpression); + return List.of(functionExpression); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java index ba8fb10a1cd..dcf1f857832 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java @@ -7,7 +7,6 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Objects; @@ -31,7 +30,7 @@ public class NotNode extends BooleanNode { @Override public List<ExpressionNode> children() { - return Collections.singletonList(value); + return List.of(value); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d1cb77fb1b4..271e367c1f4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -9,9 +9,9 @@ import com.yahoo.tensor.evaluation.TypeContext; import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -29,7 +29,7 @@ public class SerializationContext extends FunctionReferenceContext { /** Create a context for a single serialization task */ public SerializationContext() { - this(Collections.emptyList(), Collections.emptyMap(), Optional.empty(), new LinkedHashMap<>()); + this(List.of(), Map.of(), Optional.empty(), new LinkedHashMap<>()); } /** @@ -40,7 +40,7 @@ public class SerializationContext extends FunctionReferenceContext { */ public SerializationContext(Collection<ExpressionFunction> functions, Optional<TypeContext<Reference>> typeContext) { - this(functions, Collections.emptyMap(), typeContext, new LinkedHashMap<>()); + this(functions, Map.of(), typeContext, new LinkedHashMap<>()); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index b3f2f265900..202dbebc311 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -19,7 +19,6 @@ import com.yahoo.tensor.functions.TensorFunction; import com.yahoo.tensor.functions.ToStringContext; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; @@ -238,7 +237,7 @@ public class TensorFunctionNode extends CompositeNode { .map(ExpressionTensorFunction::new) .collect(Collectors.toList()); else - return Collections.emptyList(); + return List.of(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/UnpackBitsNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/UnpackBitsNode.java index 81ad09dd880..0a0f48df823 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/UnpackBitsNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/UnpackBitsNode.java @@ -11,7 +11,6 @@ import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.Objects; @@ -53,7 +52,7 @@ public class UnpackBitsNode extends CompositeNode { @Override public List<ExpressionNode> children() { - return Collections.singletonList(input); + return List.of(input); } private static record Meta(TensorType outputType, TensorType outputDenseType, String unpackDimension) {} diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 97aa42f79c9..51f433bf67d 100755..100644 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -24,7 +24,6 @@ import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.*; import com.yahoo.tensor.functions.*; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Arrays; import java.util.ArrayList; @@ -893,7 +892,7 @@ List<String> bracedIdentifierList() : String element; } { - ( element = identifier() { return Collections.singletonList(element); } ) + ( element = identifier() { return List.of(element); } ) | ( <LBRACE> list = identifierList() <RBRACE> { return list; } ) } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupTestCase.java index e306d27b6ee..60309db9787 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Identifiable; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -56,7 +55,7 @@ public class GroupTestCase { group.addOrderBy(bar, false); assertEquals(2, group.getOrderByExpressions().size()); assertSame(bar, group.getOrderByExpressions().get(1)); - assertEquals(Arrays.asList(1, -2), group.getOrderByIndexes()); + assertEquals(List.of(1, -2), group.getOrderByIndexes()); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java index 66b8a07ac95..a0e6fd32ddc 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingTestCase.java @@ -1,14 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.aggregation; -import com.yahoo.searchlib.expression.FloatResultNode; import com.yahoo.searchlib.expression.NullResultNode; import com.yahoo.searchlib.expression.StringBucketResultNode; import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @@ -30,7 +28,7 @@ public class GroupingTestCase { assertEquals(9, grouping.getId()); Grouping other = new Grouping(6); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setId(9); assertEquals(grouping, other); @@ -46,7 +44,7 @@ public class GroupingTestCase { assertTrue(grouping.getAll()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setAll(true); assertEquals(grouping, other); @@ -62,7 +60,7 @@ public class GroupingTestCase { assertEquals(69, grouping.getTopN()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setTopN(69); assertEquals(grouping, other); @@ -78,7 +76,7 @@ public class GroupingTestCase { assertEquals(69, grouping.getFirstLevel()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setFirstLevel(69); assertEquals(grouping, other); @@ -94,7 +92,7 @@ public class GroupingTestCase { assertEquals(69, grouping.getLastLevel()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setLastLevel(69); assertEquals(grouping, other); @@ -117,7 +115,7 @@ public class GroupingTestCase { assertEquals(root, grouping.getRoot()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.setRoot(root); assertEquals(grouping, other); @@ -128,7 +126,7 @@ public class GroupingTestCase { @Test public void requireThatLevelAccessorsWork() { Grouping grouping = new Grouping(); - assertEquals(Collections.emptyList(), grouping.getLevels()); + assertEquals(List.of(), grouping.getLevels()); try { grouping.addLevel(null); fail(); @@ -137,10 +135,10 @@ public class GroupingTestCase { } GroupingLevel level = new GroupingLevel(); grouping.addLevel(level); - assertEquals(Arrays.asList(level), grouping.getLevels()); + assertEquals(List.of(level), grouping.getLevels()); Grouping other = new Grouping(); - assertFalse(grouping.equals(other)); + assertNotEquals(grouping, other); other.addLevel(level); assertEquals(grouping, other); @@ -155,8 +153,8 @@ public class GroupingTestCase { @Test public void requireThatEqualsIsImplemented() { - assertFalse(new Grouping().equals(new Object())); - assertTrue(new Grouping().equals(new Grouping())); + assertNotEquals(new Grouping(), new Object()); + assertEquals(new Grouping(), new Grouping()); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java index 610352ff236..838a4d738ac 100755..100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.searchlib.expression.MultiplyFunctionNode; import com.yahoo.searchlib.expression.StringResultNode; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -725,20 +724,20 @@ public class MergeTestCase { } private static void assertMerge(Grouping request, Group lhs, Group rhs, Group expect) { - assertMerge(Arrays.asList(request.clone().setRoot(lhs.clone()), + assertMerge(List.of(request.clone().setRoot(lhs.clone()), request.clone().setRoot(rhs.clone())), expect); } private static void assertMerge(Grouping request, Group a, Group b, Group c, Group expect) { - assertMerge(Arrays.asList(request.clone().setRoot(a.clone()), + assertMerge(List.of(request.clone().setRoot(a.clone()), request.clone().setRoot(b.clone()), request.clone().setRoot(c.clone())), expect); } private static void assertMerge(Grouping lhs, Grouping rhs, Group expect) { - assertMerge(Arrays.asList(lhs, rhs), expect); + assertMerge(List.of(lhs, rhs), expect); } private static void assertMerge(List<Grouping> groupingList, Group expect) { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java index d5269611b53..8e54c4598a0 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java @@ -4,7 +4,6 @@ package com.yahoo.searchlib.aggregation.hll; import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -98,7 +97,7 @@ public class NormalSketchTest { NormalSketch sketch = new NormalSketch(10); // Aggregate multiple values - sketch.aggregate(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); + sketch.aggregate(List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); for (int i = 0; i < 10; i++) { assertBucketEquals(sketch, i, 23); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchUtils.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchUtils.java index 9bec86c928b..5bf5294a828 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchUtils.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchUtils.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.aggregation.hll; -import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; @@ -16,13 +16,13 @@ public class SketchUtils { public static SparseSketch createSparseSketch(Integer... values) { SparseSketch sketch = new SparseSketch(); - sketch.aggregate(Arrays.asList(values)); + sketch.aggregate(List.of(values)); return sketch; } public static NormalSketch createNormalSketch(Integer... values) { NormalSketch sketch = new NormalSketch(); - sketch.aggregate(Arrays.asList(values)); + sketch.aggregate(List.of(values)); return sketch; } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java index c765714a4ab..45d809dcd8a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.searchlib.expression; import org.junit.Test; import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertArrayEquals; @@ -18,7 +17,7 @@ import static org.junit.Assert.assertTrue; public class IntegerResultNodeTestCase extends ResultNodeTest { List<NumericResultNode> getResultNodes(long startvalue) { - return Arrays.asList(new Int8ResultNode((byte)startvalue), + return List.of(new Int8ResultNode((byte)startvalue), new Int16ResultNode((short)startvalue), new Int32ResultNode((int)startvalue), new IntegerResultNode(startvalue)); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java index 3c224d06cc3..1346577569b 100755..100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.ObjectDumper; import org.junit.Test; + import java.util.Arrays; import static org.junit.Assert.assertEquals; @@ -18,38 +19,46 @@ public class ObjectVisitorTestCase { assertDump("test: <NULL>\n", null); assertDump("test: 1\n", 1); assertDump("test: 'foo'\n", "foo"); - assertDump("test: List {\n" + - " [0]: 'foo'\n" + - " [1]: 69\n" + - " [2]: <NULL>\n" + - "}\n", + assertDump(""" + test: List { + [0]: 'foo' + [1]: 69 + [2]: <NULL> + } + """, Arrays.asList("foo", 69, null)); - assertDump("test: String[] {\n" + - " [0]: 'foo'\n" + - " [1]: 'bar'\n" + - " [2]: 'baz'\n" + - "}\n", + assertDump(""" + test: String[] { + [0]: 'foo' + [1]: 'bar' + [2]: 'baz' + } + """, new String[] { "foo", "bar", "baz" }); - assertDump("test: IntegerResultNode {\n" + - " classId: 16491\n" + - " value: 5\n" + - "}\n", + assertDump(""" + test: IntegerResultNode { + classId: 16491 + value: 5 + } + """, new IntegerResultNode(5)); - assertDump("test: FixedWidthBucketFunctionNode {\n" + - " classId: 16461\n" + - " result: <NULL>\n" + - " args: List {\n" + - " [0]: AttributeNode {\n" + - " classId: 16439\n" + - " result: <NULL>\n" + - " attribute: 'foo'\n" + - " }\n" + - " }\n" + - " width: IntegerResultNode {\n" + - " classId: 16491\n" + - " value: 5\n" + - " }\n" + - "}\n", + assertDump(""" + test: FixedWidthBucketFunctionNode { + classId: 16461 + result: <NULL> + args: List { + [0]: AttributeNode { + classId: 16439 + result: <NULL> + attribute: 'foo' + } + } + width: IntegerResultNode { + classId: 16491 + value: 5 + } + } + """, new FixedWidthBucketFunctionNode(new IntegerResultNode(5), new AttributeNode("foo"))); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/TimeStampFunctionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/TimeStampFunctionTestCase.java index 69cea09df81..188603749e2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/TimeStampFunctionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/TimeStampFunctionTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchlib.expression; import org.junit.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; @@ -17,7 +17,7 @@ public class TimeStampFunctionTestCase { public void requireThatAccessorsWork() { ExpressionNode arg = new AttributeNode("foo"); for (TimeStampFunctionNode.TimePart part : TimeStampFunctionNode.TimePart.values()) { - for (Boolean gmt : Arrays.asList(true, false)) { + for (Boolean gmt : List.of(true, false)) { TimeStampFunctionNode node = new TimeStampFunctionNode(arg, part, gmt); assertSame(arg, node.getArg()); assertEquals(part, node.getTimePart()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java index 8de36ee1ec1..0af105885ec 100755..100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java @@ -8,7 +8,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.Reader; import java.io.StringReader; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -22,37 +21,37 @@ public class FeatureListTestCase { @Test public void requireThatFeatureListFromStringWorks() throws ParseException { assertFromString("attribute(foo).out", - Arrays.asList("attribute(foo).out")); + List.of("attribute(foo).out")); assertFromString("attribute(foo).out attribute ( bar ) . out", - Arrays.asList("attribute(foo).out", "attribute(bar).out")); + List.of("attribute(foo).out", "attribute(bar).out")); assertFromString("foo\n bar\n \t \t \n baz \n", - Arrays.asList("foo", "bar", "baz")); + List.of("foo", "bar", "baz")); assertFromString("attribute attribute(foo) attribute(foo).out attribute(bar).out.out", - Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + List.of("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); } @Test public void requireThatFeatureListFromReaderWorks() throws ParseException { assertFromReader(new StringReader("attribute(foo).out"), - Arrays.asList("attribute(foo).out")); + List.of("attribute(foo).out")); assertFromReader(new StringReader("attribute(foo).out attribute ( bar ) . out"), - Arrays.asList("attribute(foo).out", "attribute(bar).out")); + List.of("attribute(foo).out", "attribute(bar).out")); assertFromReader(new StringReader("foo\n bar\n \t \t \n baz \n"), - Arrays.asList("foo", "bar", "baz")); + List.of("foo", "bar", "baz")); assertFromReader(new StringReader("attribute attribute(foo) attribute(foo).out attribute(bar).out.out"), - Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + List.of("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); } @Test public void requireThatFeatureListFromFileWorks() throws ParseException, FileNotFoundException { assertFromFile(new File("src/test/files/features01.expression"), - Arrays.asList("attribute(foo).out")); + List.of("attribute(foo).out")); assertFromFile(new File("src/test/files/features02.expression"), - Arrays.asList("attribute(foo).out", "attribute(bar).out")); + List.of("attribute(foo).out", "attribute(bar).out")); assertFromFile(new File("src/test/files/features03.expression"), - Arrays.asList("foo", "bar", "baz")); + List.of("foo", "bar", "baz")); assertFromFile(new File("src/test/files/features04.expression"), - Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + List.of("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); } public void assertFromString(String input, List<String> expected) throws ParseException { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 8d60f893c7c..13714af8324 100755..100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -121,28 +120,28 @@ public class RankingExpressionTestCase { @Test public void testSerialization() throws ParseException { List<ExpressionFunction> functions = new ArrayList<>(); - functions.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); - functions.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); - functions.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); + functions.add(new ExpressionFunction("foo", List.of("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); + functions.add(new ExpressionFunction("bar", List.of("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); + functions.add(new ExpressionFunction("baz", List.of("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); functions.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); - assertSerialization(Arrays.asList( + assertSerialization(List.of( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(foo@af74e3fd9070bd18.a368ed0a5ba3a5d0) * rankingExpression(foo@dbab346efdad5362.e5c39e42ebd91c30)", "min(5,pow(rankingExpression(foo@d1d1417259cdc651.573bbcd4be18f379),2))", "min(6,pow(7,2))", "min(1,pow(2,2))", "min(3,pow(4,2))", "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", functions); - assertSerialization(Arrays.asList( + assertSerialization(List.of( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(bar@af74e3fd9070bd18.a368ed0a5ba3a5d0)", "min(1,pow(2,2))", "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", functions); - assertSerialization(Arrays.asList( + assertSerialization(List.of( "rankingExpression(baz@e2dc17a89864aed0.12232eb692c6c502)", "min(1,pow(2,2))", "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) / rankingExpression(bar@e2dc17a89864aed0.12232eb692c6c502)", "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", functions); - assertSerialization(Arrays.asList( + assertSerialization(List.of( "rankingExpression(cox)", "10 + 8 * 1977"), "cox", functions ); @@ -237,8 +236,8 @@ public class RankingExpressionTestCase { String expRhs = "(rankingExpression(log10tweetage) * rankingExpression(log10tweetage) * " + "rankingExpression(log10tweetage)) + 5.0 * attribute(ythl)"; - assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions); - assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions); + assertSerialization(List.of(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions); + assertSerialization(List.of(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ArgumentsTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ArgumentsTestCase.java index df91c78c1b4..af6e4f1b9a9 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ArgumentsTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ArgumentsTestCase.java @@ -3,8 +3,8 @@ package com.yahoo.searchlib.rankingexpression.rule; import org.junit.Test; -import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @@ -23,7 +23,7 @@ public class ArgumentsTestCase { var foo = new ReferenceNode("foo"); var bar = new ReferenceNode("bar"); - args = new Arguments(Arrays.asList(foo, bar)); + args = new Arguments(List.of(foo, bar)); assertEquals(2, args.expressions().size()); assertSame(foo, args.expressions().get(0)); assertSame(bar, args.expressions().get(1)); @@ -31,12 +31,12 @@ public class ArgumentsTestCase { @Test public void requireThatHashCodeAndEqualsWork() { - Arguments arg1 = new Arguments(Arrays.asList(new ReferenceNode("foo"), new ReferenceNode("bar"))); - Arguments arg2 = new Arguments(Arrays.asList(new ReferenceNode("foo"), new ReferenceNode("bar"))); - Arguments arg3 = new Arguments(Arrays.asList(new ReferenceNode("foo"))); + Arguments arg1 = new Arguments(List.of(new ReferenceNode("foo"), new ReferenceNode("bar"))); + Arguments arg2 = new Arguments(List.of(new ReferenceNode("foo"), new ReferenceNode("bar"))); + Arguments arg3 = new Arguments(List.of(new ReferenceNode("foo"))); assertEquals(arg1.hashCode(), arg2.hashCode()); - assertTrue(arg1.equals(arg2)); - assertFalse(arg2.equals(arg3)); + assertEquals(arg1, arg2); + assertNotEquals(arg2, arg3); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNodeTestCase.java index 7d0e0d6da84..10a37ff28df 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNodeTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchlib.rankingexpression.rule; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -15,7 +14,7 @@ public class ReferenceNodeTestCase { @Test public void requireThatAccessorsWork() { - ReferenceNode node = new ReferenceNode("foo", Arrays.asList(new ReferenceNode("bar"), new ReferenceNode("baz")), "cox"); + ReferenceNode node = new ReferenceNode("foo", List.of(new ReferenceNode("bar"), new ReferenceNode("baz")), "cox"); assertEquals("foo", node.getName()); List<ExpressionNode> args = node.getArguments().expressions(); assertEquals(2, args.size()); @@ -23,10 +22,10 @@ public class ReferenceNodeTestCase { assertEquals(new ReferenceNode("baz"), args.get(1)); assertEquals("cox", node.getOutput()); - node = node.setArguments(Arrays.<ExpressionNode>asList(new ReferenceNode("bar@"))); + node = node.setArguments(List.of(new ReferenceNode("bar@"))); assertEquals(new ReferenceNode("bar@"), node.getArguments().expressions().get(0)); - node = node.setArguments(Arrays.<ExpressionNode>asList(new ReferenceNode("baz$"))); + node = node.setArguments(List.of(new ReferenceNode("baz$"))); assertEquals(new ReferenceNode("baz$"), node.getArguments().expressions().get(0)); node = node.setOutput("cox'"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java index 0f367bb5425..7736b0e9ac7 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java @@ -12,7 +12,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.NegativeNode; import org.junit.Test; -import java.util.Collections; +import java.util.Map; import static org.junit.Assert.*; @@ -24,7 +24,7 @@ public class SimplifierTestCase { @Test public void testSimplify() throws ParseException { Simplifier s = new Simplifier(); - TransformContext c = new TransformContext(Collections.emptyMap(), new MapTypeContext()); + TransformContext c = new TransformContext(Map.of(), new MapTypeContext()); assertEquals("a + b", s.transform(new RankingExpression("a + b"), c).toString()); assertEquals("6.5", s.transform(new RankingExpression("1.0 + 2.0 + 3.5"), c).toString()); assertEquals("6.5", s.transform(new RankingExpression("1.0 + ( 2.0 + 3.5 )"), c).toString()); @@ -48,7 +48,7 @@ public class SimplifierTestCase { @Test public void testNaNExpression() throws ParseException { Simplifier s = new Simplifier(); - TransformContext c = new TransformContext(Collections.emptyMap(), new MapTypeContext()); + TransformContext c = new TransformContext(Map.of(), new MapTypeContext()); assertEquals("0 / 0", s.transform(new RankingExpression("0/0"), c).toString()); assertEquals("1 + 0.0 / 0.0", s.transform(new RankingExpression("1 + (1-1)/(2-2)"), c).toString()); } @@ -56,7 +56,7 @@ public class SimplifierTestCase { @Test public void testSimplifyComplexExpression() throws ParseException { RankingExpression initial = new RankingExpression("sqrt(if (if (INFERRED * 0.9 < INFERRED, GMP, (1 + 1.1) * INFERRED) < INFERRED * INFERRED - INFERRED, if (GMP < 85.80799542793133 * GMP, INFERRED, if (GMP < GMP, tanh(INFERRED), log(76.89956221113943))), tanh(tanh(INFERRED))) * sqrt(sqrt(GMP + INFERRED)) * GMP ) + 13.5 * (1 - GMP) * pow(GMP * 0.1, 2 + 1.1 * 0)"); - TransformContext c = new TransformContext(Collections.emptyMap(), new MapTypeContext()); + TransformContext c = new TransformContext(Map.of(), new MapTypeContext()); RankingExpression simplified = new Simplifier().transform(initial, c); Context context = new MapContext(); @@ -81,7 +81,7 @@ public class SimplifierTestCase { @Test public void testParenthesisPreservation() throws ParseException { Simplifier s = new Simplifier(); - TransformContext c = new TransformContext(Collections.emptyMap(), new MapTypeContext()); + TransformContext c = new TransformContext(Map.of(), new MapTypeContext()); CompositeNode transformed = (CompositeNode)s.transform(new RankingExpression("a + (b + c) / 100000000.0"), c).getRoot(); assertEquals("a + (b + c) / 1.0E8", transformed.toString()); } @@ -89,7 +89,7 @@ public class SimplifierTestCase { @Test public void testOptimizingNegativeConstants() throws ParseException { Simplifier s = new Simplifier(); - TransformContext c = new TransformContext(Collections.emptyMap(), new MapTypeContext()); + TransformContext c = new TransformContext(Map.of(), new MapTypeContext()); assertEquals("-3", s.transform(new RankingExpression("-3"), c).toString()); assertEquals("-9.0", s.transform(new RankingExpression("-3 + -6"), c).toString()); assertEquals("-a", s.transform(new RankingExpression("-a"), c).toString()); diff --git a/searchlib/src/tests/aggregator/perdocexpr_test.cpp b/searchlib/src/tests/aggregator/perdocexpr_test.cpp index 908e50ad4d2..e9f0981739c 100644 --- a/searchlib/src/tests/aggregator/perdocexpr_test.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr_test.cpp @@ -604,6 +604,7 @@ getVespaChecksumV2(const std::string& ymumid, int fid, const std::string& flags_ sizeof(networkFid)+ new_flags_str.length(); + // GNU extension: Variable-length automatic array unsigned char buffer[length]; memset(buffer, 0x00, length); memcpy(buffer, ymumid.c_str(), ymumid.length()); diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp index 8ba8c62c5ff..55810c02550 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp @@ -120,11 +120,12 @@ struct MatchStats { template <bool collect_matches> void -brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, + bool cased, bool prefix_match, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - FuzzyMatcher matcher(target, 2, prefix_size, cased); + FuzzyMatcher matcher(target, 2, prefix_size, cased, prefix_match); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -144,11 +145,12 @@ brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumS template <bool collect_matches> void -dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, + bool cased, bool prefix_match, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, prefix_match, LevenshteinDfa::DfaType::Explicit); Utf8Reader reader(vespalib::stringref(target.data(), target.size())); std::string target_copy; Utf8Writer<std::string> writer(target_copy); @@ -187,11 +189,12 @@ dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& st template <bool collect_matches> void -dfa_fuzzy_match_in_dictionary_no_skip(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary_no_skip(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, + bool cased, bool prefix_match, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, prefix_match, LevenshteinDfa::DfaType::Explicit); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -247,20 +250,23 @@ struct DfaFuzzyMatcherTest : public ::testing::TestWithParam<TestParam> { updater.commit(); store.freeze_dictionary(); } - void expect_prefix_matches(std::string_view target, uint32_t prefix_size, const StringVector& exp_matches) { + void expect_prefix_matches(std::string_view target, uint32_t prefix_size, bool prefix_match, const StringVector& exp_matches) { MatchStats stats; StringVector brute_force_matches; StringVector dfa_matches; StringVector dfa_no_skip_matches; bool cased = GetParam()._cased; SCOPED_TRACE(target); - brute_force_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, brute_force_matches); - dfa_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, stats, dfa_matches); - dfa_fuzzy_match_in_dictionary_no_skip<true>(target, store, prefix_size, cased, stats, dfa_no_skip_matches); + brute_force_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, prefix_match, stats, brute_force_matches); + dfa_fuzzy_match_in_dictionary<true>(target, store, prefix_size, cased, prefix_match, stats, dfa_matches); + dfa_fuzzy_match_in_dictionary_no_skip<true>(target, store, prefix_size, cased, prefix_match, stats, dfa_no_skip_matches); EXPECT_EQ(exp_matches, brute_force_matches); EXPECT_EQ(exp_matches, dfa_matches); EXPECT_EQ(exp_matches, dfa_no_skip_matches); } + void expect_prefix_matches(std::string_view target, uint32_t prefix_size, const StringVector& exp_matches) { + expect_prefix_matches(target, prefix_size, false, exp_matches); + } void expect_matches(std::string_view target, const StringVector& exp_matches) { expect_prefix_matches(target, 0, exp_matches); } @@ -284,11 +290,12 @@ TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) expect_matches("forcecast", {"forecast"}); } -TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_lock_length) { bool cased = GetParam()._cased; StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", - "for", "forbid", "force", "ford", "forearm", "forecast", "forest", "H", "HA", "h", "ha", char_from_u8(u8"Ørn"), char_from_u8(u8"øre"), char_from_u8(u8"Ã…s"), char_from_u8(u8"Ã¥s")}; + "for", "forbid", "force", "ford", "forearm", "forecast", "forest", "H", "HA", "h", "ha", + char_from_u8(u8"Ørn"), char_from_u8(u8"øre"), char_from_u8(u8"Ã…s"), char_from_u8(u8"Ã¥s")}; populate_dictionary(words); expect_prefix_matches("a", 1, {}); expect_prefix_matches("b", 1, {"bob"}); @@ -321,6 +328,25 @@ TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) } } +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_matching) { + // We include the empty string to make "everything matches"-checks easier since + // the dictionary implicitly returns such an empty string sentinel already. + StringVector words = {"", "board", "boat", "bob", "door", "food", "foot", "football", "foothill", + "for", "forbid", "force", "ford", "forearm", "forecast", "forest"}; + populate_dictionary(words); + expect_prefix_matches("bo", 0, true, words); // matches everything + expect_prefix_matches("bo", 2, true, {"board", "boat", "bob"}); + expect_prefix_matches("boar", 0, true, {"board", "boat", "bob", "door", "for", "forbid", + "force", "ford", "forearm", "forecast", "forest"}); + expect_prefix_matches("board", 0, true, {"board", "boat", "ford"}); + expect_prefix_matches("footb", 0, true, {"food", "foot", "football", "foothill", "forbid"}); + expect_prefix_matches("footba", 0, true, {"foot", "football", "foothill"}); + expect_prefix_matches("footbal", 0, true, {"football", "foothill"}); + expect_prefix_matches("football", 0, true, {"football", "foothill"}); + expect_prefix_matches("footballs", 0, true, {"football"}); + expect_prefix_matches("z", 1, true, {}); +} + void benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool cased, bool dfa_algorithm) { @@ -329,9 +355,9 @@ benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDicti for (size_t i = 0; i < std::min(words_to_match, dict.size()); ++i) { const auto& entry = dict[i]; if (dfa_algorithm) { - dfa_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); + dfa_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, false, stats, dummy); } else { - brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, stats, dummy); + brute_force_fuzzy_match_in_dictionary<false>(entry.first, store, 0, cased, false, stats, dummy); } } std::cout << (dfa_algorithm ? "DFA:" : "Brute force:") << " samples=" << stats.samples << ", avg_matches=" << stats.avg_matches() << ", avg_seeks=" << stats.avg_seeks() << ", avg_elapsed_ms=" << stats.avg_elapsed_ms() << std::endl; diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute_test.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute_test.cpp index 48270694394..d67757a3811 100644 --- a/searchlib/src/tests/attribute/extendattributes/extendattribute_test.cpp +++ b/searchlib/src/tests/attribute/extendattributes/extendattribute_test.cpp @@ -224,7 +224,7 @@ void ExtendAttributeTest::testExtendRaw(AttributeVector& attr) void ExtendAttributeTest::testExtendTensor(AttributeVector& attr) { - std::vector<double> empty_cells{0.0, 0.0}; + std::vector<double> empty_cells{}; std::vector<double> spec0_dense_cells{1.0, 2.0}; std::vector<double> spec0_mixed_cells0{3.0, 4.0}; std::vector<double> spec0_mixed_cells1{5.0, 6.0}; diff --git a/searchlib/src/tests/diskindex/pagedict4/.gitignore b/searchlib/src/tests/diskindex/pagedict4/.gitignore index 2381ed57229..8aa95a29b63 100644 --- a/searchlib/src/tests/diskindex/pagedict4/.gitignore +++ b/searchlib/src/tests/diskindex/pagedict4/.gitignore @@ -3,3 +3,4 @@ Makefile pagedict4_test fakedict.* searchlib_pagedict4_test_app +/long_words_dir/ diff --git a/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt b/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt index 6be544db829..34114e195bc 100644 --- a/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt +++ b/searchlib/src/tests/diskindex/pagedict4/CMakeLists.txt @@ -16,3 +16,11 @@ vespa_add_executable(searchlib_pagedict4_hugeword_cornercase_test_app TEST searchlib ) vespa_add_test(NAME searchlib_pagedict4_hugeword_cornercase_test_app COMMAND searchlib_pagedict4_hugeword_cornercase_test_app) + +vespa_add_executable(searchlib_pagedict4_long_words_test_app TEST + SOURCES + pagedict4_long_words_test.cpp + DEPENDS + searchlib_test + searchlib +) diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4_long_words_test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4_long_words_test.cpp new file mode 100644 index 00000000000..dba7980a4b7 --- /dev/null +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4_long_words_test.cpp @@ -0,0 +1,131 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/searchlib/diskindex/pagedict4file.h> +#include <vespa/searchlib/diskindex/pagedict4randread.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <filesystem> + +using search::diskindex::PageDict4FileSeqRead; +using search::diskindex::PageDict4FileSeqWrite; +using search::diskindex::PageDict4RandRead; +using search::index::DummyFileHeaderContext; +using search::index::PostingListCounts; +using search::index::PostingListOffsetAndCounts; +using search::index::PostingListParams; + + +namespace { + +vespalib::string test_dir("long_words_dir"); +vespalib::string dict(test_dir + "/dict"); + +PostingListCounts make_counts() +{ + PostingListCounts counts; + counts._bitLength = 100; + counts._numDocs = 1; + counts._segments.clear(); + return counts; +} + +vespalib::string +make_word(int i) +{ + vespalib::asciistream os; + vespalib::string word(5_Ki, 'a'); + os << vespalib::setfill('0') << vespalib::setw(8) << i; + word.append(os.str()); + return word; +} + +} + +/* + * A long word that don't fit into a 4 KiB 'page' causes a fallback to + * overflow handling where the word is put in the .ssdat file. + * + * Many long words causes excessive growth of the .ssdat file, with + * overflow potentials when the whole file is read into a buffer. + * + * 4 GiB size: Overflow in ComprFileReadBase::ReadComprBuffer for expression + * readUnits * cbuf.getUnitSize() when both are 32-bits. + * Testable by setting num_words to 900_Ki + * + * 16 GiB size: Overflow in ComprFileReadBase::ReadComprBuffer when + * readUnits is 32-bit signed. + * Some overflows in ComprFileDecodeContext API. + * Overflow in DecodeContext64Base::getBitPos + * Testable by setting num_words to 4_Mi + * + * 32 GiB size: Overflow when calling ComprFileReadContext::allocComprBuf when + * comprBufSize is 32-bit unsigned. + * Overflow in DecodeContext64Base::setEnd. + * Testable by setting num_words to 9_Mi + * + * These overflows are fixed. + */ +TEST(PageDict4LongWordsTest, test_many_long_words) +{ + int num_words = 9_Mi; + auto counts = make_counts(); + std::filesystem::remove_all(std::filesystem::path(test_dir)); + std::filesystem::create_directories(std::filesystem::path(test_dir)); + + auto dw = std::make_unique<PageDict4FileSeqWrite>(); + DummyFileHeaderContext file_header_context; + PostingListParams params; + search::TuneFileSeqWrite tune_file_write; + params.set("numWordIds", num_words); + params.set("minChunkDocs", 256_Ki); + dw->setParams(params); + EXPECT_TRUE(dw->open(dict, tune_file_write, file_header_context)); + for (int i = 0; i < num_words; ++i) { + auto word = make_word(i); + dw->writeWord(word, counts); + } + EXPECT_TRUE(dw->close()); + dw.reset(); + + auto drr = std::make_unique<PageDict4RandRead>(); + search::TuneFileRandRead tune_file_rand_read; + EXPECT_TRUE(drr->open(dict, tune_file_rand_read)); + PostingListOffsetAndCounts offset_and_counts; + uint64_t exp_offset = 0; + uint64_t exp_acc_num_docs = 0; + for (int i = 0; i < num_words; ++i) { + auto word = make_word(i); + uint64_t check_word_num = 0; + EXPECT_TRUE(drr->lookup(word, check_word_num, offset_and_counts)); + EXPECT_EQ(i + 1, (int) check_word_num); + EXPECT_EQ(exp_offset, offset_and_counts._offset); + EXPECT_EQ(exp_acc_num_docs, offset_and_counts._accNumDocs); + EXPECT_EQ(counts, offset_and_counts._counts); + exp_offset += offset_and_counts._counts._bitLength; + exp_acc_num_docs += offset_and_counts._counts._numDocs; + } + EXPECT_TRUE(drr->close()); + drr.reset(); + + auto dr = std::make_unique<PageDict4FileSeqRead>(); + search::TuneFileSeqRead tune_file_read; + EXPECT_TRUE(dr->open(dict, tune_file_read)); + vespalib::string check_word; + PostingListCounts check_counts; + for (int i = 0; i < num_words; ++i) { + uint64_t check_word_num = 0; + check_word.clear(); + dr->readWord(check_word, check_word_num, check_counts); + EXPECT_EQ(i + 1, (int) check_word_num); + EXPECT_EQ(make_word(i), check_word); + EXPECT_EQ(counts, check_counts); + } + EXPECT_TRUE(dr->close()); + dr.reset(); + + std::filesystem::remove_all(std::filesystem::path(test_dir)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4_test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4_test.cpp index 951d6f61980..3b7ec00211d 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4_test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4_test.cpp @@ -15,8 +15,9 @@ #include <vespa/searchlib/diskindex/pagedict4randread.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/util/signalhandler.h> -#include <sstream> #include <cinttypes> +#include <optional> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP("pagedict4test"); @@ -357,6 +358,7 @@ checkCounts(const std::string &word, void testWords(const std::string &logname, vespalib::Rand48 &rnd, + std::optional<uint32_t> mmap_file_size_threshold, uint64_t numWordIds, uint32_t tupleCount, uint32_t chunkSize, @@ -495,7 +497,14 @@ testWords(const std::string &logname, LOG(info, "%s: pagedict4 written", logname.c_str()); } { - std::unique_ptr<DictionaryFileSeqRead> dr(new PageDict4FileSeqRead); + std::unique_ptr<DictionaryFileSeqRead> dr; + { + auto my_dr = std::make_unique<PageDict4FileSeqRead>(); + if (mmap_file_size_threshold.has_value()) { + my_dr->set_mmap_file_size_threshold(mmap_file_size_threshold.value()); + } + dr = std::move(my_dr); + } search::TuneFileSeqRead tuneFileRead; bool openres = dr->open("fakedict", @@ -535,7 +544,14 @@ testWords(const std::string &logname, LOG(info, "%s: pagedict4 seqverify OK", logname.c_str()); } { - std::unique_ptr<DictionaryFileRandRead> drr(new PageDict4RandRead); + std::unique_ptr<DictionaryFileRandRead> drr; + { + auto my_drr = std::make_unique<PageDict4RandRead>(); + if (mmap_file_size_threshold.has_value()) { + my_drr->set_mmap_file_size_threshold(mmap_file_size_threshold.value()); + } + drr = std::move(my_drr); + } search::TuneFileRandRead tuneFileRead; bool openres = drr->open("fakedict", tuneFileRead); @@ -649,46 +665,50 @@ testWords(const std::string &logname, void PageDict4TestApp::testWords() { - ::testWords("smallchunkwordsempty", _rnd, + ::testWords("smallchunkwordsempty", _rnd, std::nullopt, 1000000, 0, 64, 80, 72, 64, false, false, false); - ::testWords("smallchunkwordsempty2", _rnd, + ::testWords("smallchunkwordsempty2", _rnd, std::nullopt, 0, 0, 64, 80, 72, 64, false, false, false); - ::testWords("smallchunkwords", _rnd, + ::testWords("smallchunkwords", _rnd, std::nullopt, 1000000, 100, 64, 80, 72, 64, false, false, false); - ::testWords("smallchunkwordswithemptyword", _rnd, + ::testWords("smallchunkwordswithemptyword", _rnd, std::nullopt, 1000000, 100, 64, 80, 72, 64, true, false, false); - ::testWords("smallchunkwordswithcommonfirstword", _rnd, + ::testWords("smallchunkwordswithcommonfirstword", _rnd, std::nullopt, 1000000, 100, 64, 80, 72, 64, false, true, false); - ::testWords("smallchunkwordswithcommonemptyfirstword", _rnd, + ::testWords("smallchunkwordswithcommonemptyfirstword", _rnd, std::nullopt, 1000000, 100, 64, 80, 72, 64, true, true, false); - ::testWords("smallchunkwordswithcommonlastword", _rnd, + ::testWords("smallchunkwordswithcommonlastword", _rnd, std::nullopt, 1000000, 100, 64, 80, 72, 64, false, false, true); -#if 1 - ::testWords("smallchunkwords2", _rnd, + ::testWords("smallchunkwords2", _rnd, std::nullopt, 1000000, _stress ? 10000 : 100, 64, 80, 72, 64, _emptyWord, _firstWordForcedCommon, _lastWordForcedCommon); -#endif -#if 1 - ::testWords("stdwords", _rnd, + ::testWords("stdwords", _rnd, std::nullopt, 1000000, _stress ? 10000 : 100, 262144, 80, 72, 64, _emptyWord, _firstWordForcedCommon, _lastWordForcedCommon); -#endif + ::testWords("stdwordsnommapssdat", _rnd, 500_Mi, + 1000000, 100, + 262144, 80, 72, 64, + _emptyWord, _firstWordForcedCommon, _lastWordForcedCommon); + ::testWords("stdwordsmmapssdat", _rnd, 1, + 1000000, 100, + 262144, 80, 72, 64, + _emptyWord, _firstWordForcedCommon, _lastWordForcedCommon); } int main(int argc, char **argv) { diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp index 158899e22da..5b3e3c356ac 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride_test.cpp @@ -14,6 +14,8 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/objects/nbostream.h> + using namespace search::features; using namespace search::fef::test; diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 815ff3bf1fb..fb89f2ef061 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -37,7 +37,7 @@ struct MyRangeTerm : InitTerm<RangeTerm> {}; struct MyStringTerm : InitTerm<StringTerm> {}; struct MySubstrTerm : InitTerm<SubstringTerm> {}; struct MySuffixTerm : InitTerm<SuffixTerm> {}; -struct MyFuzzyTerm : FuzzyTerm { MyFuzzyTerm(): FuzzyTerm("term", "view", 0, Weight(0), 2, 0) {} }; +struct MyFuzzyTerm : FuzzyTerm { MyFuzzyTerm(): FuzzyTerm("term", "view", 0, Weight(0), 2, 0, false) {} }; struct MyWeakAnd : WeakAnd { MyWeakAnd() : WeakAnd(1234, "view") {} }; struct MyWeightedSetTerm : WeightedSetTerm { MyWeightedSetTerm() : WeightedSetTerm(0, "view", 0, Weight(42)) {} }; struct MyDotProduct : DotProduct { MyDotProduct() : DotProduct(0, "view", 0, Weight(42)) {} }; diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index b0c9c4e570e..bfce382b684 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -88,7 +88,7 @@ TEST("requireThatAllNodesCanBeVisited") { checkVisit<NearestNeighborTerm>(new SimpleNearestNeighborTerm("query_tensor", "doc_tensor", 0, Weight(0), 123, true, 321, 100100.25)); checkVisit<TrueQueryNode>(new SimpleTrue()); checkVisit<FalseQueryNode>(new SimpleFalse()); - checkVisit<FuzzyTerm>(new SimpleFuzzyTerm("t", "field", 0, Weight(0), 2, 0)); + checkVisit<FuzzyTerm>(new SimpleFuzzyTerm("t", "field", 0, Weight(0), 2, 0, false)); checkVisit<InTerm>(new SimpleInTerm(std::make_unique<StringTermVector>(0), MultiTerm::Type::STRING, "field", 0, Weight(0))); } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index be795c8c94b..c70fe5b510b 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -117,7 +117,7 @@ Node::UP createQueryTree() { builder.add_true_node(); builder.add_false_node(); } - builder.addFuzzyTerm(str[5], view[5], id[5], weight[5], 3, 1); + builder.addFuzzyTerm(str[5], view[5], id[5], weight[5], 3, 1, false); } Node::UP node = builder.build(); ASSERT_TRUE(node.get()); @@ -326,8 +326,8 @@ void checkQueryTreeTypes(Node *node) { auto* fuzzy_term = as_node<FuzzyTerm>(and_node->getChildren()[12]); EXPECT_TRUE(checkTerm(fuzzy_term, str[5], view[5], id[5], weight[5])); - EXPECT_EQUAL(3u, fuzzy_term->getMaxEditDistance()); - EXPECT_EQUAL(1u, fuzzy_term->getPrefixLength()); + EXPECT_EQUAL(3u, fuzzy_term->max_edit_distance()); + EXPECT_EQUAL(1u, fuzzy_term->prefix_lock_length()); } struct AbstractTypes { @@ -452,8 +452,9 @@ struct MyTrue : TrueQueryNode {}; struct MyFalse : FalseQueryNode {}; struct MyFuzzyTerm : FuzzyTerm { MyFuzzyTerm(const Type &t, const string &f, int32_t i, Weight w, - uint32_t m, uint32_t p) - : FuzzyTerm(t, f, i, w, m, p) { + uint32_t m, uint32_t p, bool prefix_match) + : FuzzyTerm(t, f, i, w, m, p, prefix_match) + { } }; struct MyInTerm : InTerm { @@ -645,23 +646,27 @@ TEST("require that All Range Syntaxes Work") { EXPECT_TRUE(range2 == range_term->getTerm()); } -TEST("require that fuzzy node can be created") { - QueryBuilder<SimpleQueryNodeTypes> builder; - builder.addFuzzyTerm("term", "view", 0, Weight(0), 3, 1); - Node::UP node = builder.build(); +TEST("fuzzy node can be created") { + for (bool prefix_match : {false, true}) { + QueryBuilder<SimpleQueryNodeTypes> builder; + builder.addFuzzyTerm("term", "view", 0, Weight(0), 3, 1, prefix_match); + Node::UP node = builder.build(); - string stackDump = StackDumpCreator::create(*node); - { - SimpleQueryStackDumpIterator iterator(stackDump); - Node::UP new_node = QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); - FuzzyTerm *fuzzy_node = as_node<FuzzyTerm>(new_node.get()); - EXPECT_EQUAL(3u, fuzzy_node->getMaxEditDistance()); - EXPECT_EQUAL(1u, fuzzy_node->getPrefixLength()); - } - { - search::QueryTermSimple::UP queryTermSimple = search::QueryTermDecoder::decodeTerm(stackDump); - EXPECT_EQUAL(3u, queryTermSimple->getFuzzyMaxEditDistance()); - EXPECT_EQUAL(1u, queryTermSimple->getFuzzyPrefixLength()); + string stackDump = StackDumpCreator::create(*node); + { + SimpleQueryStackDumpIterator iterator(stackDump); + Node::UP new_node = QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator); + auto *fuzzy_node = as_node<FuzzyTerm>(new_node.get()); + EXPECT_EQUAL(3u, fuzzy_node->max_edit_distance()); + EXPECT_EQUAL(1u, fuzzy_node->prefix_lock_length()); + EXPECT_EQUAL(prefix_match, fuzzy_node->prefix_match()); + } + { + search::QueryTermSimple::UP queryTermSimple = search::QueryTermDecoder::decodeTerm(stackDump); + EXPECT_EQUAL(3u, queryTermSimple->fuzzy_max_edit_distance()); + EXPECT_EQUAL(1u, queryTermSimple->fuzzy_prefix_lock_length()); + EXPECT_EQUAL(prefix_match, queryTermSimple->fuzzy_prefix_match()); + } } } diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 6ec1ffd460e..485410e0eba 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -39,7 +39,7 @@ public: return mixChildrenFields(); } - void sort(Children &children, bool, bool) const override { + void sort(Children &children, InFlow) const override { std::sort(children.begin(), children.end(), TieredGreaterEstimate()); } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 72dd2b5a4ad..bddc9f92111 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -869,7 +869,7 @@ TEST("AND_NOT AND AND_NOT collapsing into full source blender optimization") { TEST("test single child optimization") { InvalidSelector selector; //------------------------------------------------------------------------- - Blueprint::UP top = make::ANDNOT().add(make::AND().add(make::RANK().add(make::OR().add(make::SB(selector).source(2).add(make::RANK().leaf(42)))))); + Blueprint::UP top = make::ANDNOT().add(make::AND().add(make::RANK().add(make::OR().add(make::WEAKAND(100).add(make::SB(selector).source(2).add(make::RANK().leaf(42))))))); //------------------------------------------------------------------------- Blueprint::UP expect = make::SB(selector).source(2).leaf(42); //------------------------------------------------------------------------- diff --git a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp index 181c591ab20..57fddb0a819 100644 --- a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp +++ b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp @@ -7,6 +7,7 @@ constexpr size_t loop_cnt = 64; constexpr size_t max_work = 1; // 500'000'000; +constexpr bool dump_unexpected = false; constexpr bool verbose = false; using namespace search::queryeval; @@ -40,7 +41,7 @@ double dual_ordered_cost_of(const std::vector<FlowStats> &data, InFlow in_flow, any_flow.update_cost(total_cost, child_cost); any_flow.add(item.estimate); } - EXPECT_EQ(total_cost, result); + EXPECT_DOUBLE_EQ(total_cost, result); return result; } @@ -174,7 +175,7 @@ void verify_flow(auto flow, const std::vector<double> &est_list, const std::vect AnyFlow any_flow = AnyFlow::create<decltype(flow)>(InFlow(flow.strict(), flow.flow())); ASSERT_EQ(est_list.size() + 1, expect.size()); for (size_t i = 0; i < est_list.size(); ++i) { - EXPECT_EQ(any_flow.flow(), flow.flow()); + EXPECT_DOUBLE_EQ(any_flow.flow(), flow.flow()); EXPECT_EQ(any_flow.strict(), flow.strict()); EXPECT_DOUBLE_EQ(flow.flow(), expect[i].flow); EXPECT_EQ(flow.strict(), expect[i].strict); @@ -182,7 +183,7 @@ void verify_flow(auto flow, const std::vector<double> &est_list, const std::vect any_flow.add(est_list[i]); flow.add(est_list[i]); } - EXPECT_EQ(any_flow.flow(), flow.flow()); + EXPECT_DOUBLE_EQ(any_flow.flow(), flow.flow()); EXPECT_EQ(any_flow.strict(), flow.strict()); EXPECT_DOUBLE_EQ(flow.flow(), expect.back().flow); EXPECT_EQ(flow.strict(), expect.back().strict); @@ -356,10 +357,10 @@ TEST(FlowTest, optimal_and_flow) { double min_cost = AndFlow::cost_of(data, strict); double max_cost = 0.0; AndFlow::sort(data, strict); - EXPECT_EQ(ordered_cost_of<AndFlow>(data, strict, false), min_cost); + EXPECT_DOUBLE_EQ(ordered_cost_of<AndFlow>(data, strict, false), min_cost); auto check = [&](const std::vector<FlowStats> &my_data) noexcept { double my_cost = ordered_cost_of<AndFlow>(my_data, strict, false); - EXPECT_LE(min_cost, my_cost); + EXPECT_LE(min_cost, my_cost + 1e-9); max_cost = std::max(max_cost, my_cost); }; each_perm(data, check); @@ -379,7 +380,7 @@ TEST(FlowTest, optimal_or_flow) { double min_cost = OrFlow::cost_of(data, strict); double max_cost = 0.0; OrFlow::sort(data, strict); - EXPECT_EQ(ordered_cost_of<OrFlow>(data, strict, false), min_cost); + EXPECT_DOUBLE_EQ(ordered_cost_of<OrFlow>(data, strict, false), min_cost); auto check = [&](const std::vector<FlowStats> &my_data) noexcept { double my_cost = ordered_cost_of<OrFlow>(my_data, strict, false); EXPECT_LE(min_cost, my_cost + 1e-9); @@ -420,198 +421,144 @@ TEST(FlowTest, optimal_and_not_flow) { } } -void test_strict_AND_sort_strategy(auto my_sort) { - re_seed(); +void test_AND_sort_strategy(auto my_sort) { const char *tags = "ABCDEFGHI"; - for (size_t child_cnt: {2, 3, 5, 7, 9}) { - size_t cnt = std::max(size_t(10), std::min(size_t(128'000), (max_work / count_perms(child_cnt)))); - if (verbose) { - fprintf(stderr, "AND/%zu: checking all permutations for %zu random cases\n", child_cnt, cnt); - } - std::vector<FlowStats> my_worst_order; - std::vector<FlowStats> best_worst_order; - auto get_tag = [&](const FlowStats &stats, const std::vector<FlowStats> &ref)noexcept->char{ - for (size_t i = 0; i < ref.size(); ++i) { - if (stats == ref[i]) { - return tags[i]; + for (InFlow in_flow: {InFlow(true), InFlow(0.5)}) { + re_seed(); + for (size_t child_cnt: {2, 3, 5, 7, 9}) { + size_t cnt = std::max(size_t(10), std::min(size_t(128'000), (max_work / count_perms(child_cnt)))); + if (verbose) { + fprintf(stderr, "%6f %s -> AND/%zu: checking all permutations for %zu random cases\n", + in_flow.rate(), in_flow.strict() ? "S" : " ", child_cnt, cnt); + } + std::vector<FlowStats> my_worst_order; + std::vector<FlowStats> best_worst_order; + auto get_tag = [&](const FlowStats &stats, const std::vector<FlowStats> &ref)noexcept->char{ + for (size_t i = 0; i < ref.size(); ++i) { + if (stats == ref[i]) { + return tags[i]; + } } - } - return 'X'; - }; - auto dump_flow = [&](const std::vector<FlowStats> &list, const std::vector<FlowStats> &ref){ - double total_cost = 0.0; - auto flow = AndFlow(true); - for (const auto &item: list) { - auto in_flow = InFlow(flow.strict(), flow.flow()); - bool strict = flow.strict() || flow::should_force_strict(item, flow.flow()); - double child_cost = flow::min_child_cost(in_flow, item, true); - fprintf(stderr, " %10f -> %c (estimate: %10f, cost: %10f, strict_cost: %10f, cross: %10f, gain: %10f, gain@est: %10f) cost: %10f%s\n", - flow.flow(), get_tag(item, ref), item.estimate, item.cost, item.strict_cost, strict_crossover(item), strict_gain(item, in_flow), strict_gain(item, item.estimate), - child_cost, strict ? " STRICT" : ""); - flow.add(item.estimate); - total_cost += child_cost; - } - EXPECT_EQ(total_cost, ordered_cost_of<AndFlow>(list, true, true)); - fprintf(stderr, " total cost: %10f\n", total_cost); - }; - auto verify_order = [&](const std::vector<FlowStats> &list){ - // check the following constraints for the given order: - // - // (1) never strict after non-strict - // (2) strict items are sorted by estimate - // (3) non-strict items are sorted by max(reduction/cost) - auto flow = AndFlow(true); - size_t strict_limit = list.size(); - auto my_cmp = flow::MinAndCost(flow::DirectAdapter()); - for (size_t i = 0; i < list.size(); ++i) { - const auto &item = list[i]; - if (i > 0) { - const auto &prev = list[i-1]; - bool strict = flow::should_force_strict(item, flow.flow()); + return 'X'; + }; + auto dump_flow = [&](const std::vector<FlowStats> &list, const std::vector<FlowStats> &ref){ + double total_cost = 0.0; + auto flow = AndFlow(in_flow); + for (const auto &item: list) { + auto child_flow = InFlow(flow.strict(), flow.flow()); + bool strict = flow.strict() || flow::should_force_strict(item, flow.flow()); + double child_cost = flow::min_child_cost(child_flow, item, true); + fprintf(stderr, " %6f %s -> %c (estimate: %10f, cost: %10f, strict_cost: %10f, cross: %10f, gain: %10f, gain@est: %10f) cost: %10f%s\n", + flow.flow(), flow.strict() ? "S" : " ", get_tag(item, ref), item.estimate, item.cost, item.strict_cost, strict_crossover(item), + strict_gain(item, child_flow), strict_gain(item, item.estimate), + child_cost, strict ? " STRICT" : ""); + flow.add(item.estimate); + total_cost += child_cost; + } + EXPECT_DOUBLE_EQ(total_cost, ordered_cost_of<AndFlow>(list, in_flow, true)); + fprintf(stderr, " total cost: %10f\n", total_cost); + }; + auto verify_order = [&](const std::vector<FlowStats> &list){ + // check the following constraints for the given order: + // + // (1) never strict after non-strict + // (2) strict items are sorted by estimate + // (3) non-strict items are sorted by max(reduction/cost) + auto flow = AndFlow(in_flow); + auto my_cmp = flow::MinAndCost(flow::DirectAdapter()); + size_t num_non_strict = 0; + FlowStats prev_strict(0.0, 0.0, 0.0); + FlowStats prev_non_strict(0.0, 0.0, 0.0); + for (const auto &item: list) { + bool strict = flow.strict() || flow::should_force_strict(item, flow.flow()); if (strict) { - if (i > strict_limit) { + if (num_non_strict > 0) { return false; // (1) - } else if (item.estimate < prev.estimate) { + } else if (item.estimate < prev_strict.estimate) { return false; // (2) } + prev_strict = item; } else { - strict_limit = std::min(i, strict_limit); - if ((strict_limit < i) && my_cmp(item, prev)) { + if (my_cmp(item, prev_non_strict)) { return false; // (3) } + ++num_non_strict; + prev_non_strict = item; } + flow.add(item.estimate); } - flow.add(item.estimate); - } - return true; - }; - double max_rel_err = 0.0; - double sum_rel_err = 0.0; - std::vector<double> errs; - errs.reserve(cnt); - auto p = [&](double arg){ - size_t idx = std::lround(arg * (errs.size() - 1)); - if (idx < errs.size()) { - return errs[idx]; - } - return errs.back(); - }; - for (size_t i = 0; i < cnt; ++i) { - auto data = gen_data(child_cnt); - double ref_est = AndFlow::estimate_of(data); - my_sort(data); - auto my_order = data; - auto best_order = my_order; - double est_cost = ordered_cost_of<AndFlow>(data, true, true); - double min_cost = est_cost; - double max_cost = est_cost; - auto check = [&](const std::vector<FlowStats> &my_data) noexcept { - double my_cost = ordered_cost_of<AndFlow>(my_data, true, true); - if (my_cost < min_cost) { - min_cost = my_cost; - best_order = my_data; - } - max_cost = std::max(max_cost, my_cost); - }; - each_perm(data, check); - double rel_err = 0.0; - double cost_range = (max_cost - min_cost); - if (cost_range > 1e-9) { - rel_err = (est_cost - min_cost) / cost_range; - } - if (rel_err > max_rel_err) { - max_rel_err = rel_err; - my_worst_order = my_order; - best_worst_order = best_order; + return true; + }; + double max_rel_err = 0.0; + double sum_rel_err = 0.0; + std::vector<double> errs; + errs.reserve(cnt); + auto p = [&](double arg){ + size_t idx = std::lround(arg * (errs.size() - 1)); + if (idx < errs.size()) { + return errs[idx]; + } + return errs.back(); + }; + for (size_t i = 0; i < cnt; ++i) { + auto data = gen_data(child_cnt); + double ref_est = AndFlow::estimate_of(data); + my_sort(data, in_flow); + auto my_order = data; + auto best_order = my_order; + double est_cost = ordered_cost_of<AndFlow>(data, in_flow, true); + double min_cost = est_cost; + double max_cost = est_cost; + auto check = [&](const std::vector<FlowStats> &my_data) noexcept { + double my_cost = ordered_cost_of<AndFlow>(my_data, in_flow, true); + if (my_cost < min_cost) { + min_cost = my_cost; + best_order = my_data; + } + max_cost = std::max(max_cost, my_cost); + }; + each_perm(data, check); + double rel_err = 0.0; + rel_err = (est_cost - min_cost) / min_cost; + if (rel_err > max_rel_err) { + max_rel_err = rel_err; + my_worst_order = my_order; + best_worst_order = best_order; + } + sum_rel_err += rel_err; + errs.push_back(rel_err); + if (dump_unexpected && !verify_order(best_order)) { + fprintf(stderr, " BEST ORDER IS UNEXPECTED:\n"); + dump_flow(best_order, best_order); + fprintf(stderr, " UNEXPECTED case, my_order:\n"); + dump_flow(my_order, best_order); + } + EXPECT_NEAR(ref_est, AndFlow::estimate_of(data), 1e-9); } - sum_rel_err += rel_err; - errs.push_back(rel_err); - if (verbose && !verify_order(best_order)) { - fprintf(stderr, " BEST ORDER IS UNEXPECTED:\n"); - dump_flow(best_order, best_order); - fprintf(stderr, " UNEXPECTED case, my_order:\n"); - dump_flow(my_order, best_order); + std::sort(errs.begin(), errs.end()); + if (verbose && !my_worst_order.empty()) { + fprintf(stderr, " worst case, best order:\n"); + dump_flow(best_worst_order, best_worst_order); + fprintf(stderr, " worst case, my order:\n"); + dump_flow(my_worst_order, best_worst_order); } - EXPECT_NEAR(ref_est, AndFlow::estimate_of(data), 1e-9); - } - std::sort(errs.begin(), errs.end()); - if (verbose && !my_worst_order.empty()) { - fprintf(stderr, " worst case, best order:\n"); - dump_flow(best_worst_order, best_worst_order); - fprintf(stderr, " worst case, my order:\n"); - dump_flow(my_worst_order, best_worst_order); + fprintf(stderr, "%6f %s -> AND/%zu: avg: %10f, p90: %10f, p99: %10f, p99.9: %10f, max: %10f\n", + in_flow.rate(), in_flow.strict() ? "S" : " ", child_cnt, (sum_rel_err / cnt), p(0.9), p(0.99), p(0.999), max_rel_err); } - fprintf(stderr, "AND/%zu: avg: %10f, p90: %10f, p99: %10f, p99.9: %10f, max: %10f\n", - child_cnt, (sum_rel_err / cnt), p(0.9), p(0.99), p(0.999), max_rel_err); } } -TEST(FlowTest, strict_and_with_allow_force_strict_basic_order) { - auto my_sort = [](auto &data){ AndFlow::sort(data, true); }; - test_strict_AND_sort_strategy(my_sort); +TEST(FlowTest, and_with_allow_force_strict_basic_order) { + auto my_sort = [](auto &data, InFlow in_flow){ AndFlow::sort(data, in_flow.strict()); }; + test_AND_sort_strategy(my_sort); } -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - for (size_t next = 1; (next + 1) < data.size(); ++next) { - auto [idx, score] = flow::select_forced_strict_and_child(flow::DirectAdapter(), data, next); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + next, pos, pos + 1); - } - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_strict_re_sorting) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - size_t strict_cnt = 1; - for (; strict_cnt < data.size(); ++strict_cnt) { - auto [idx, score] = flow::select_forced_strict_and_child(flow::DirectAdapter(), data, strict_cnt); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + strict_cnt, pos, pos + 1); - } - std::sort(data.begin(), data.begin() + strict_cnt, - [](const auto &a, const auto &b){ return (a.estimate < b.estimate); }); - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_order) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - for (size_t next = 1; next < data.size(); ++next) { - auto [idx, target, score] = flow::select_forced_strict_and_child_with_order(flow::DirectAdapter(), data, next); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + target, pos, pos + 1); - } - }; - test_strict_AND_sort_strategy(my_sort); -} - -TEST(FlowTest, strict_and_with_allow_force_strict_incremental_strict_selection_with_destructive_order) { - auto my_sort = [](auto &data) { - AndFlow::sort(data, true); - for (size_t next = 1; next < data.size(); ++next) { - auto [idx, target, score] = flow::select_forced_strict_and_child_with_destructive_order(flow::DirectAdapter(), data, next); - if (score >= 0.0) { - break; - } - auto pos = data.begin() + idx; - std::rotate(data.begin() + target, pos, pos + 1); - } +TEST(FlowTest, and_with_allow_force_strict_incremental_strict_selection_destructive_order_max_3_extra_strict) { + auto my_sort = [](auto &data, InFlow in_flow) { + AndFlow::sort(data, in_flow.strict()); + AndFlow::reorder_for_extra_strictness(data, in_flow, 3); }; - test_strict_AND_sort_strategy(my_sort); + test_AND_sort_strategy(my_sort); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt b/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt index 872fb4ca6ca..dadd06ee7cd 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/iterator_benchmark/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_iterator_benchmark_test_app TEST SOURCES + intermediate_blueprint_factory.cpp attribute_ctx_builder.cpp benchmark_blueprint_factory.cpp common.cpp diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.cpp index 6a2b306522d..5a0bda49b98 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.cpp @@ -18,30 +18,71 @@ namespace { template <typename AttributeType, bool is_string, bool is_multivalue> void +update_attribute(AttributeType& attr, uint32_t docid, uint32_t value) +{ + if constexpr (is_string) { + if constexpr (is_multivalue) { + attr.append(docid, std::to_string(value), random_int(1, 100)); + } else { + attr.update(docid, std::to_string(value)); + } + } else { + if constexpr (is_multivalue) { + attr.append(docid, value, random_int(1, 100)); + } else { + attr.update(docid, value); + } + } +} + +template <typename AttributeType, bool is_string, bool is_multivalue> +void populate_attribute(AttributeType& attr, uint32_t docid_limit, const HitSpecs& hit_specs) { for (auto spec : hit_specs) { auto docids = random_docids(docid_limit, spec.num_hits); docids->foreach_truebit([&](uint32_t docid) { - if constexpr (is_string) { - if constexpr (is_multivalue) { - attr.append(docid, std::to_string(spec.term_value), random_int(1, 100)); - } else { - attr.update(docid, std::to_string(spec.term_value)); - } - } else { - if constexpr (is_multivalue) { - attr.append(docid, spec.term_value, random_int(1, 100)); - } else { - attr.update(docid, spec.term_value); - } - } + update_attribute<AttributeType, is_string, is_multivalue>(attr, docid, spec.term_value); }); } } +template <typename AttributeType, bool is_string, bool is_multivalue> +void +populate_attribute(AttributeType& attr, const std::vector<uint32_t>& values) +{ + for (uint32_t docid = 1; docid < values.size(); ++docid) { + uint32_t value = values[docid]; + if (value == 0) { + continue; + } + update_attribute<AttributeType, is_string, is_multivalue>(attr, docid, value); + } +} + +template <typename AttributeType, bool is_string, bool is_multivalue> +void +populate_attribute(AttributeType& attr, uint32_t docid_limit, const HitSpecs& hit_specs, bool disjunct_terms) +{ + if (disjunct_terms) { + // Ensure that each term in HitSpecs is matched by a disjunct (random) subset of docids. + std::vector<uint32_t> values(docid_limit, 0); + uint32_t docid = 1; + for (auto spec : hit_specs) { + assert((docid + spec.num_hits) <= docid_limit); + std::fill_n(values.begin() + docid, spec.num_hits, spec.term_value); + docid += spec.num_hits; + } + std::shuffle(values.begin() + 1, values.end(), get_gen()); + populate_attribute<AttributeType, is_string, is_multivalue>(attr, values); + } else { + // For each term in HitSpecs we draw a new random set of docids that will match this term value. + populate_attribute<AttributeType, is_string, is_multivalue>(attr, docid_limit, hit_specs); + } +} + AttributeVector::SP -make_attribute(const Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs) +make_attribute(const Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs, bool disjunct_terms) { auto attr = AttributeFactory::createAttribute(field_name, cfg); attr->addReservedDoc(); @@ -52,16 +93,16 @@ make_attribute(const Config& cfg, vespalib::stringref field_name, uint32_t num_d if (attr->isStringType()) { auto& real = dynamic_cast<StringAttribute&>(*attr); if (is_multivalue) { - populate_attribute<StringAttribute, true, true>(real, docid_limit, hit_specs); + populate_attribute<StringAttribute, true, true>(real, docid_limit, hit_specs, disjunct_terms); } else { - populate_attribute<StringAttribute, true, false>(real, docid_limit, hit_specs); + populate_attribute<StringAttribute, true, false>(real, docid_limit, hit_specs, disjunct_terms); } } else { auto& real = dynamic_cast<IntegerAttribute&>(*attr); if (is_multivalue) { - populate_attribute<IntegerAttribute, false, true>(real, docid_limit, hit_specs); + populate_attribute<IntegerAttribute, false, true>(real, docid_limit, hit_specs, disjunct_terms); } else { - populate_attribute<IntegerAttribute, false, false>(real, docid_limit, hit_specs); + populate_attribute<IntegerAttribute, false, false>(real, docid_limit, hit_specs, disjunct_terms); } } attr->commit(true); @@ -90,9 +131,9 @@ AttributeContextBuilder::AttributeContextBuilder() } void -AttributeContextBuilder::add(const search::attribute::Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs) +AttributeContextBuilder::add(const search::attribute::Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs, bool disjunct_terms) { - auto attr = make_attribute(cfg, field_name, num_docs, hit_specs); + auto attr = make_attribute(cfg, field_name, num_docs, hit_specs, disjunct_terms); _ctx->add(std::move(attr)); } diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.h b/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.h index e4a58c91668..7e5236e43be 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.h +++ b/searchlib/src/tests/queryeval/iterator_benchmark/attribute_ctx_builder.h @@ -19,7 +19,7 @@ private: public: AttributeContextBuilder(); - void add(const search::attribute::Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs); + void add(const search::attribute::Config& cfg, vespalib::stringref field_name, uint32_t num_docs, const HitSpecs& hit_specs, bool disjunct_terms); std::unique_ptr<BenchmarkSearchable> build(); }; diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.cpp index 516a819aae8..504ad057d3d 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.cpp @@ -42,11 +42,11 @@ calc_hits_per_term(uint32_t num_docs, double op_hit_ratio, uint32_t children, Qu } std::unique_ptr<BenchmarkSearchable> -make_searchable(const FieldConfig& cfg, uint32_t num_docs, const HitSpecs& hit_specs) +make_searchable(const FieldConfig& cfg, uint32_t num_docs, const HitSpecs& hit_specs, bool disjunct_terms) { if (cfg.is_attr()) { AttributeContextBuilder builder; - builder.add(cfg.attr_cfg(), field_name, num_docs, hit_specs); + builder.add(cfg.attr_cfg(), field_name, num_docs, hit_specs, disjunct_terms); return builder.build(); } else { uint32_t docid_limit = num_docs + 1; @@ -159,14 +159,17 @@ private: public: MyFactory(const FieldConfig& field_cfg, QueryOperator query_op, uint32_t num_docs, uint32_t default_values_per_document, - double op_hit_ratio, uint32_t children); + double op_hit_ratio, uint32_t children, bool disjunct_children); std::unique_ptr<Blueprint> make_blueprint() override; + vespalib::string get_name(Blueprint& blueprint) const override { + return get_class_name(blueprint); + } }; MyFactory::MyFactory(const FieldConfig& field_cfg, QueryOperator query_op, uint32_t num_docs, uint32_t default_values_per_document, - double op_hit_ratio, uint32_t children) + double op_hit_ratio, uint32_t children, bool disjunct_children) : _query_op(query_op), _docid_limit(num_docs + 1), _terms(), @@ -174,9 +177,17 @@ MyFactory::MyFactory(const FieldConfig& field_cfg, QueryOperator query_op, { uint32_t hits_per_term = calc_hits_per_term(num_docs, op_hit_ratio, children, query_op); HitSpecs hit_specs(55555); - hit_specs.add(default_values_per_document, num_docs); + if (!disjunct_children) { + hit_specs.add(default_values_per_document, num_docs); + } _terms = hit_specs.add(children, hits_per_term); - _searchable = make_searchable(field_cfg, num_docs, hit_specs); + if (disjunct_children && default_values_per_document != 0) { + // This ensures that the remaining docids are populated with a "default value". + // Only a single default value is supported. + uint32_t op_num_hits = num_docs * op_hit_ratio; + hit_specs.add(1, num_docs - op_num_hits); + } + _searchable = make_searchable(field_cfg, num_docs, hit_specs, disjunct_children); } std::unique_ptr<Blueprint> @@ -190,9 +201,9 @@ MyFactory::make_blueprint() std::unique_ptr<BenchmarkBlueprintFactory> make_blueprint_factory(const FieldConfig& field_cfg, QueryOperator query_op, uint32_t num_docs, uint32_t default_values_per_document, - double op_hit_ratio, uint32_t children) + double op_hit_ratio, uint32_t children, bool disjunct_children) { - return std::make_unique<MyFactory>(field_cfg, query_op, num_docs, default_values_per_document, op_hit_ratio, children); + return std::make_unique<MyFactory>(field_cfg, query_op, num_docs, default_values_per_document, op_hit_ratio, children, disjunct_children); } } diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.h b/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.h index d3e529fcd65..2a90dbbbef8 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.h +++ b/searchlib/src/tests/queryeval/iterator_benchmark/benchmark_blueprint_factory.h @@ -16,11 +16,12 @@ class BenchmarkBlueprintFactory { public: virtual ~BenchmarkBlueprintFactory() = default; virtual std::unique_ptr<Blueprint> make_blueprint() = 0; + virtual vespalib::string get_name(Blueprint& blueprint) const = 0; }; std::unique_ptr<BenchmarkBlueprintFactory> make_blueprint_factory(const FieldConfig& field_cfg, QueryOperator query_op, uint32_t num_docs, uint32_t default_values_per_document, - double op_hit_ratio, uint32_t children); + double op_hit_ratio, uint32_t children, bool disjunct_children); } diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/common.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/common.cpp index f17403bd33a..1db9cd58d46 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/common.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/common.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "common.h" -#include <random> +#include <vespa/searchlib/queryeval/blueprint.h> #include <sstream> using search::attribute::CollectionType; @@ -20,7 +20,11 @@ to_string(const Config& attr_config) oss << col_type.asString() << "<" << basic_type.asString() << ">"; } if (attr_config.fastSearch()) { - oss << "(fs)"; + oss << "(fs"; + if (attr_config.getIsFilter()) { + oss << ",rf"; + } + oss << ")"; } return oss.str(); } @@ -43,12 +47,46 @@ to_string(QueryOperator query_op) namespace { +std::string +delete_substr_from(const std::string& source, const std::string& substr) +{ + std::string res = source; + auto i = res.find(substr); + while (i != std::string::npos) { + res.erase(i, substr.length()); + i = res.find(substr, i); + } + return res; +} + +} + +vespalib::string +get_class_name(const auto& obj) +{ + auto res = obj.getClassName(); + res = delete_substr_from(res, "search::attribute::"); + res = delete_substr_from(res, "search::queryeval::"); + res = delete_substr_from(res, "vespalib::btree::"); + res = delete_substr_from(res, "search::"); + res = delete_substr_from(res, "vespalib::"); + res = delete_substr_from(res, "anonymous namespace"); + return res; +} + +template vespalib::string get_class_name<Blueprint>(const Blueprint& obj); +template vespalib::string get_class_name<SearchIterator>(const SearchIterator& obj); + +namespace { + // TODO: Make seed configurable. constexpr uint32_t default_seed = 1234; std::mt19937 gen(default_seed); } +std::mt19937& get_gen() { return gen; } + BitVector::UP random_docids(uint32_t docid_limit, uint32_t count) { diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/common.h b/searchlib/src/tests/queryeval/iterator_benchmark/common.h index 45fd82b091c..6341b16e96a 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/common.h +++ b/searchlib/src/tests/queryeval/iterator_benchmark/common.h @@ -5,6 +5,7 @@ #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/common/bitvector.h> +#include <random> #include <variant> namespace search::queryeval::test { @@ -78,6 +79,10 @@ public: auto end() const { return _specs.end(); } }; +vespalib::string get_class_name(const auto& obj); + +std::mt19937& get_gen(); + BitVector::UP random_docids(uint32_t docid_limit, uint32_t count); int32_t random_int(int32_t a, int32_t b); diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp new file mode 100644 index 00000000000..8591ec1415d --- /dev/null +++ b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.cpp @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "intermediate_blueprint_factory.h" +#include <vespa/searchlib/queryeval/intermediate_blueprints.h> +#include <iomanip> +#include <sstream> + +namespace search::queryeval::test { + +template <typename BlueprintType> +char +IntermediateBlueprintFactory<BlueprintType>::child_name(void* blueprint) const +{ + auto itr = _child_names.find(blueprint); + if (itr != _child_names.end()) { + return itr->second; + } + return '?'; +} + +template <typename BlueprintType> +IntermediateBlueprintFactory<BlueprintType>::IntermediateBlueprintFactory(vespalib::stringref name) + : _name(name), + _children(), + _child_names() +{ +} + +template <typename BlueprintType> +IntermediateBlueprintFactory<BlueprintType>::~IntermediateBlueprintFactory() = default; + +template <typename BlueprintType> +std::unique_ptr<Blueprint> +IntermediateBlueprintFactory<BlueprintType>::make_blueprint() +{ + auto res = std::make_unique<BlueprintType>(); + _child_names.clear(); + char name = 'A'; + for (const auto& factory : _children) { + auto child = factory->make_blueprint(); + _child_names[child.get()] = name++; + res->addChild(std::move(child)); + } + return res; +} + +template <typename BlueprintType> +vespalib::string +IntermediateBlueprintFactory<BlueprintType>::get_name(Blueprint& blueprint) const +{ + auto* intermediate = blueprint.asIntermediate(); + if (intermediate != nullptr) { + std::ostringstream oss; + bool first = true; + oss << _name << "["; + for (size_t i = 0; i < intermediate->childCnt(); ++i) { + auto* child = &intermediate->getChild(i); + oss << (first ? "" : ",") << child_name(child) << "."; + if (child->strict()) { + oss << "s(" << std::setw(6) << std::setprecision(3) << child->strict_cost() << ")"; + } else { + oss << "n(" << std::setw(6) << std::setprecision(3) << child->cost() << ")"; + } + first = false; + } + oss << "]"; + return oss.str(); + } + return get_class_name(blueprint); +} + +template class IntermediateBlueprintFactory<AndBlueprint>; + +AndBlueprintFactory::AndBlueprintFactory() + : IntermediateBlueprintFactory<AndBlueprint>("AND") +{} + +} + diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h new file mode 100644 index 00000000000..6f7fe4f9ee7 --- /dev/null +++ b/searchlib/src/tests/queryeval/iterator_benchmark/intermediate_blueprint_factory.h @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "benchmark_blueprint_factory.h" +#include <vespa/searchlib/queryeval/intermediate_blueprints.h> +#include <unordered_map> + +namespace search::queryeval::test { + +/** + * Factory that creates an IntermediateBlueprint (of the given type) with children created by the given factories. + */ +template <typename BlueprintType> +class IntermediateBlueprintFactory : public BenchmarkBlueprintFactory { +private: + vespalib::string _name; + std::vector<std::shared_ptr<BenchmarkBlueprintFactory>> _children; + std::unordered_map<void*, char> _child_names; + + char child_name(void* blueprint) const; + +public: + IntermediateBlueprintFactory(vespalib::stringref name); + ~IntermediateBlueprintFactory(); + void add_child(std::shared_ptr<BenchmarkBlueprintFactory> child) { + _children.push_back(std::move(child)); + } + std::unique_ptr<Blueprint> make_blueprint() override; + vespalib::string get_name(Blueprint& blueprint) const override; +}; + +class AndBlueprintFactory : public IntermediateBlueprintFactory<AndBlueprint> { +public: + AndBlueprintFactory(); +}; + +} + diff --git a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp index b08fde50d7c..f4a1ade8a66 100644 --- a/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/iterator_benchmark/iterator_benchmark_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "intermediate_blueprint_factory.h" #include "benchmark_blueprint_factory.h" #include "common.h" #include <vespa/searchlib/fef/matchdata.h> @@ -25,31 +26,47 @@ using vespalib::make_string_short::fmt; const vespalib::string field_name = "myfield"; double budget_sec = 1.0; +enum class PlanningAlgo { + Order, + Estimate, + Cost, + CostForceStrict +}; + +vespalib::string +to_string(PlanningAlgo algo) +{ + switch (algo) { + case PlanningAlgo::Order: return "ordr"; + case PlanningAlgo::Estimate: return "esti"; + case PlanningAlgo::Cost: return "cost"; + case PlanningAlgo::CostForceStrict: return "forc"; + } + return "unknown"; +} + struct BenchmarkResult { double time_ms; uint32_t seeks; uint32_t hits; FlowStats flow; double actual_cost; - double alt_cost; vespalib::string iterator_name; vespalib::string blueprint_name; - BenchmarkResult() : BenchmarkResult(0, 0, 0, {0, 0, 0}, 0, 0, "", "") {} - BenchmarkResult(double time_ms_in, uint32_t seeks_in, uint32_t hits_in, FlowStats flow_in, double actual_cost_in, double alt_cost_in, + BenchmarkResult() : BenchmarkResult(0, 0, 0, {0, 0, 0}, 0, "", "") {} + BenchmarkResult(double time_ms_in, uint32_t seeks_in, uint32_t hits_in, FlowStats flow_in, double actual_cost_in, const vespalib::string& iterator_name_in, const vespalib::string& blueprint_name_in) : time_ms(time_ms_in), seeks(seeks_in), hits(hits_in), flow(flow_in), actual_cost(actual_cost_in), - alt_cost(alt_cost_in), iterator_name(iterator_name_in), blueprint_name(blueprint_name_in) {} ~BenchmarkResult(); double ns_per_seek() const { return (time_ms / seeks) * 1000.0 * 1000.0; } double ms_per_actual_cost() const { return (time_ms / actual_cost); } - double ms_per_alt_cost() const { return (time_ms / alt_cost); } }; BenchmarkResult::~BenchmarkResult() = default; @@ -128,36 +145,8 @@ public: Stats ms_per_actual_cost_stats() const { return calc_stats([](const auto& res){ return res.ms_per_actual_cost(); }); } - Stats ms_per_alt_cost_stats() const { - return calc_stats([](const auto& res){ return res.ms_per_alt_cost(); }); - } }; -std::string -delete_substr_from(const std::string& source, const std::string& substr) -{ - std::string res = source; - auto i = res.find(substr); - while (i != std::string::npos) { - res.erase(i, substr.length()); - i = res.find(substr, i); - } - return res; -} - -vespalib::string -get_class_name(const auto& obj) -{ - auto res = obj.getClassName(); - res = delete_substr_from(res, "search::attribute::"); - res = delete_substr_from(res, "search::queryeval::"); - res = delete_substr_from(res, "vespalib::btree::"); - res = delete_substr_from(res, "search::"); - res = delete_substr_from(res, "vespalib::"); - res = delete_substr_from(res, "anonymous namespace"); - return res; -} - struct MatchLoopContext { Blueprint::UP blueprint; MatchData::UP match_data; @@ -180,12 +169,37 @@ struct MatchLoopContext { MatchLoopContext::~MatchLoopContext() = default; +Blueprint::Options +to_sort_options(PlanningAlgo algo) +{ + Blueprint::Options opts; + if (algo == PlanningAlgo::Order) { + opts.keep_order(true); + } else if (algo == PlanningAlgo::Cost) { + opts.sort_by_cost(true); + } else if (algo == PlanningAlgo::CostForceStrict) { + opts.sort_by_cost(true).allow_force_strict(true); + } + return opts; +} + +void +sort_blueprint(Blueprint& blueprint, InFlow in_flow, uint32_t docid_limit, Blueprint::Options opts) +{ + auto opts_guard = blueprint.bind_opts(opts); + blueprint.setDocIdLimit(docid_limit); + blueprint.each_node_post_order([docid_limit](Blueprint &bp){ + bp.update_flow_stats(docid_limit); + }); + blueprint.sort(in_flow); +} + MatchLoopContext -make_match_loop_context(BenchmarkBlueprintFactory& factory, InFlow in_flow, uint32_t docid_limit) +make_match_loop_context(BenchmarkBlueprintFactory& factory, InFlow in_flow, uint32_t docid_limit, PlanningAlgo algo) { auto blueprint = factory.make_blueprint(); assert(blueprint); - blueprint->basic_plan(in_flow, docid_limit); + sort_blueprint(*blueprint, in_flow, docid_limit, to_sort_options(algo)); blueprint->fetchPostings(ExecuteInfo::FULL); // Note: All blueprints get the same TermFieldMatchData instance. // This is OK as long as we don't do unpacking and only use 1 thread. @@ -197,13 +211,13 @@ make_match_loop_context(BenchmarkBlueprintFactory& factory, InFlow in_flow, uint template <bool do_unpack> BenchmarkResult -strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit) +strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, PlanningAlgo algo) { BenchmarkTimer timer(budget_sec); uint32_t hits = 0; MatchLoopContext ctx; while (timer.has_budget()) { - ctx = make_match_loop_context(factory, true, docid_limit); + ctx = make_match_loop_context(factory, true, docid_limit, algo); auto* itr = ctx.iterator.get(); timer.before(); hits = 0; @@ -222,12 +236,12 @@ strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit) timer.after(); } FlowStats flow(ctx.blueprint->estimate(), ctx.blueprint->cost(), ctx.blueprint->strict_cost()); - return {timer.min_time() * 1000.0, hits + 1, hits, flow, flow.strict_cost, flow.strict_cost, get_class_name(*ctx.iterator), get_class_name(*ctx.blueprint)}; + return {timer.min_time() * 1000.0, hits + 1, hits, flow, flow.strict_cost, get_class_name(*ctx.iterator), factory.get_name(*ctx.blueprint)}; } template <bool do_unpack> BenchmarkResult -non_strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, double filter_hit_ratio, bool force_strict) +non_strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, double filter_hit_ratio, bool force_strict, PlanningAlgo algo) { BenchmarkTimer timer(budget_sec); uint32_t seeks = 0; @@ -237,7 +251,7 @@ non_strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, doub uint32_t docid_skip = 1.0 / filter_hit_ratio; MatchLoopContext ctx; while (timer.has_budget()) { - ctx = make_match_loop_context(factory, InFlow(force_strict, filter_hit_ratio), docid_limit); + ctx = make_match_loop_context(factory, InFlow(force_strict, filter_hit_ratio), docid_limit, algo); auto* itr = ctx.iterator.get(); timer.before(); seeks = 0; @@ -256,29 +270,31 @@ non_strict_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, doub } FlowStats flow(ctx.blueprint->estimate(), ctx.blueprint->cost(), ctx.blueprint->strict_cost()); double actual_cost = flow.cost * filter_hit_ratio; - // This is an attempt to calculate an alternative actual cost for strict / posting list iterators that are used in a non-strict context. - double alt_cost = flow.strict_cost + 0.5 * filter_hit_ratio; - return {timer.min_time() * 1000.0, seeks, hits, flow, actual_cost, alt_cost, get_class_name(*ctx.iterator), get_class_name(*ctx.blueprint)}; + return {timer.min_time() * 1000.0, seeks, hits, flow, actual_cost, get_class_name(*ctx.iterator), factory.get_name(*ctx.blueprint)}; } BenchmarkResult -benchmark_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, bool strict_context, bool force_strict, bool unpack_iterator, double filter_hit_ratio) +benchmark_search(BenchmarkBlueprintFactory& factory, uint32_t docid_limit, bool strict_context, bool force_strict, bool unpack_iterator, double filter_hit_ratio, PlanningAlgo algo) { if (strict_context) { if (unpack_iterator) { - return strict_search<true>(factory, docid_limit); + return strict_search<true>(factory, docid_limit, algo); } else { - return strict_search<false>(factory, docid_limit); + return strict_search<false>(factory, docid_limit, algo); } } else { if (unpack_iterator) { - return non_strict_search<true>(factory, docid_limit, filter_hit_ratio, force_strict); + return non_strict_search<true>(factory, docid_limit, filter_hit_ratio, force_strict, algo); } else { - return non_strict_search<false>(factory, docid_limit, filter_hit_ratio, force_strict); + return non_strict_search<false>(factory, docid_limit, filter_hit_ratio, force_strict, algo); } } } + + + + //----------------------------------------------------------------------------- double est_forced_strict_cost(double estimate, double strict_cost, double rate) { @@ -351,12 +367,12 @@ void analyze_crossover(BenchmarkBlueprintFactory &fixed, std::function<std::uniq auto a = first.make_blueprint(); a->basic_plan(true, docid_limit); double est_a = a->estimate(); - double a_ms = benchmark_search(first, docid_limit, true, false, false, 1.0).time_ms; - double b_ms = benchmark_search(last, docid_limit, false, false, false, est_a).time_ms; + double a_ms = benchmark_search(first, docid_limit, true, false, false, 1.0, PlanningAlgo::Cost).time_ms; + double b_ms = benchmark_search(last, docid_limit, false, false, false, est_a, PlanningAlgo::Cost).time_ms; if (!allow_force_strict) { return Sample(a_ms + b_ms); } - double c_ms = benchmark_search(last, docid_limit, false, true, false, est_a).time_ms; + double c_ms = benchmark_search(last, docid_limit, false, true, false, est_a, PlanningAlgo::Cost).time_ms; if (c_ms < b_ms) { return Sample(a_ms + c_ms, a_ms + b_ms, true); } @@ -413,32 +429,30 @@ to_string(bool val) void print_result_header() { - std::cout << "| chn | f_ratio | o_ratio | a_ratio | f.est | f.cost | f.scost | hits | seeks | time_ms | act_cost | alt_cost | ns_per_seek | ms_per_act_cost | ms_per_alt_cost | iterator | blueprint |" << std::endl; + std::cout << "| chn | f_ratio | o_ratio | a_ratio | f.est | f.cost | f.scost | hits | seeks | time_ms | act_cost | ns_per_seek | ms_per_act_cost | iterator | blueprint |" << std::endl; } void print_result(const BenchmarkResult& res, uint32_t children, double op_hit_ratio, double filter_hit_ratio, uint32_t num_docs) { std::cout << std::fixed << std::setprecision(5) - << "| " << std::setw(4) << children + << "| " << std::setw(5) << children << " | " << std::setw(7) << filter_hit_ratio << " | " << std::setw(7) << op_hit_ratio << " | " << std::setw(7) << ((double) res.hits / (double) num_docs) << " | " << std::setw(6) << res.flow.estimate << std::setprecision(4) - << " | " << std::setw(7) << res.flow.cost + << " | " << std::setw(9) << res.flow.cost << " | " << std::setw(7) << res.flow.strict_cost << " | " << std::setw(8) << res.hits << " | " << std::setw(8) << res.seeks << std::setprecision(3) << " | " << std::setw(8) << res.time_ms << std::setprecision(4) - << " | " << std::setw(8) << res.actual_cost - << " | " << std::setw(8) << res.alt_cost + << " | " << std::setw(9) << res.actual_cost << std::setprecision(2) << " | " << std::setw(11) << res.ns_per_seek() << " | " << std::setw(15) << res.ms_per_actual_cost() - << " | " << std::setw(15) << res.ms_per_alt_cost() << " | " << res.iterator_name << " | " << res.blueprint_name << " |" << std::endl; } @@ -449,8 +463,7 @@ print_result(const BenchmarkCaseResult& result) std::cout << std::fixed << std::setprecision(3) << "summary: time_ms=" << result.time_ms_stats().to_string() << std::endl << " ns_per_seek=" << result.ns_per_seek_stats().to_string() << std::endl - << " ms_per_act_cost=" << result.ms_per_actual_cost_stats().to_string() << std::endl - << " ms_per_alt_cost=" << result.ms_per_alt_cost_stats().to_string() << std::endl << std::endl; + << " ms_per_act_cost=" << result.ms_per_actual_cost_stats().to_string() << std::endl << std::endl; } struct BenchmarkCase { @@ -534,6 +547,7 @@ struct BenchmarkCaseSetup { std::vector<uint32_t> child_counts; std::vector<double> filter_hit_ratios; uint32_t default_values_per_document; + bool disjunct_children; double filter_crossover_factor; BenchmarkCaseSetup(uint32_t num_docs_in, const BenchmarkCase& bcase_in, @@ -545,6 +559,7 @@ struct BenchmarkCaseSetup { child_counts(child_counts_in), filter_hit_ratios({1.0}), default_values_per_document(0), + disjunct_children(false), filter_crossover_factor(0.0) {} ~BenchmarkCaseSetup() {} @@ -561,6 +576,7 @@ struct BenchmarkSetup { bool force_strict; bool unpack_iterator; uint32_t default_values_per_document; + bool disjunct_children; double filter_crossover_factor; BenchmarkSetup(uint32_t num_docs_in, const std::vector<FieldConfig>& field_cfgs_in, @@ -578,6 +594,7 @@ struct BenchmarkSetup { force_strict(false), unpack_iterator(false), default_values_per_document(0), + disjunct_children(false), filter_crossover_factor(0.0) {} BenchmarkSetup(uint32_t num_docs_in, @@ -592,6 +609,7 @@ struct BenchmarkSetup { res.bcase.force_strict = force_strict; res.bcase.unpack_iterator = unpack_iterator; res.default_values_per_document = default_values_per_document; + res.disjunct_children = disjunct_children; if (!bcase.strict_context) { // Simulation of a filter is only relevant in a non-strict context. res.filter_hit_ratios = filter_hit_ratios; @@ -617,11 +635,11 @@ run_benchmark_case(const BenchmarkCaseSetup& setup) for (uint32_t children : setup.child_counts) { auto factory = make_blueprint_factory(setup.bcase.field_cfg, setup.bcase.query_op, setup.num_docs, setup.default_values_per_document, - op_hit_ratio, children); + op_hit_ratio, children, setup.disjunct_children); for (double filter_hit_ratio : setup.filter_hit_ratios) { if (filter_hit_ratio * setup.filter_crossover_factor <= op_hit_ratio) { auto res = benchmark_search(*factory, setup.num_docs + 1, - setup.bcase.strict_context, setup.bcase.force_strict, setup.bcase.unpack_iterator, filter_hit_ratio); + setup.bcase.strict_context, setup.bcase.force_strict, setup.bcase.unpack_iterator, filter_hit_ratio, PlanningAlgo::Cost); print_result(res, children, op_hit_ratio, filter_hit_ratio, setup.num_docs); result.add(res); } @@ -656,11 +674,152 @@ run_benchmarks(const BenchmarkSetup& setup) print_summary(summary); } +//--------------------------------------------------------------------------------------- +// Tools for benchmarking root intermediate blueprints with configurable children setups. +//--------------------------------------------------------------------------------------- + +void +print_intermediate_blueprint_result_header(size_t children) +{ + // This matches the naming scheme in IntermediateBlueprintFactory. + char name = 'A'; + for (size_t i = 0; i < children; ++i) { + std::cout << "| " << name++ << ".ratio "; + } + std::cout << "| flow.cost | flow.scost | flow.est | ratio | hits | seeks | ms_per_cost | time_ms | algo | blueprint |" << std::endl; +} + +void +print_intermediate_blueprint_result(const BenchmarkResult& res, const std::vector<double>& children_ratios, PlanningAlgo algo, uint32_t num_docs) +{ + std::cout << std::fixed << std::setprecision(5); + for (auto ratio : children_ratios) { + std::cout << "| " << std::setw(7) << ratio << " "; + } + std::cout << std::setprecision(5) + << "| " << std::setw(10) << res.flow.cost + << " | " << std::setw(10) << res.flow.strict_cost + << " | " << std::setw(8) << res.flow.estimate + << " | " << std::setw(7) << ((double) res.hits / (double) num_docs) + << std::setprecision(4) + << " | " << std::setw(8) << res.hits + << " | " << std::setw(8) << res.seeks + << std::setprecision(3) + << " | " << std::setw(11) << res.ms_per_actual_cost() + << " | " << std::setw(8) << res.time_ms + << " | " << to_string(algo) + << " | " << res.blueprint_name << " |" << std::endl; +} + +struct BlueprintFactorySetup { + FieldConfig field_cfg; + QueryOperator query_op; + std::vector<double> op_hit_ratios; + uint32_t children; + bool disjunct_children; + uint32_t default_values_per_document; + + BlueprintFactorySetup(const FieldConfig& field_cfg_in, QueryOperator query_op_in, const std::vector<double>& op_hit_ratios_in) + : BlueprintFactorySetup(field_cfg_in, query_op_in, op_hit_ratios_in, 1, false) + {} + BlueprintFactorySetup(const FieldConfig& field_cfg_in, QueryOperator query_op_in, const std::vector<double>& op_hit_ratios_in, + uint32_t children_in, bool disjunct_children_in) + : field_cfg(field_cfg_in), + query_op(query_op_in), + op_hit_ratios(op_hit_ratios_in), + children(children_in), + disjunct_children(disjunct_children_in), + default_values_per_document(0) + {} + ~BlueprintFactorySetup(); + std::unique_ptr<BenchmarkBlueprintFactory> make_factory(size_t num_docs, double op_hit_ratio) const { + return make_blueprint_factory(field_cfg, query_op, num_docs, default_values_per_document, op_hit_ratio, children, disjunct_children); + } + std::shared_ptr<BenchmarkBlueprintFactory> make_factory_shared(size_t num_docs, double op_hit_ratio) const { + return std::shared_ptr<BenchmarkBlueprintFactory>(make_factory(num_docs, op_hit_ratio)); + } + vespalib::string to_string() const { + return "field=" + field_cfg.to_string() + ", query=" + test::to_string(query_op) + ", children=" + std::to_string(children); + } +}; + +BlueprintFactorySetup::~BlueprintFactorySetup() = default; + +template <typename IntermediateBlueprintFactoryType> +void +run_intermediate_blueprint_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) +{ + print_intermediate_blueprint_result_header(2); + double max_speedup = 0.0; + double min_speedup = std::numeric_limits<double>::max(); + for (double b_hit_ratio: b.op_hit_ratios) { + auto b_factory = b.make_factory_shared(num_docs, b_hit_ratio); + for (double a_hit_ratio : a.op_hit_ratios) { + IntermediateBlueprintFactoryType factory; + factory.add_child(a.make_factory(num_docs, a_hit_ratio)); + factory.add_child(b_factory); + double time_ms_esti = 0.0; + for (auto algo: {PlanningAlgo::Order, PlanningAlgo::Estimate, PlanningAlgo::Cost, + PlanningAlgo::CostForceStrict}) { + auto res = benchmark_search(factory, num_docs + 1, true, false, false, 1.0, algo); + print_intermediate_blueprint_result(res, {a_hit_ratio, b_hit_ratio}, algo, num_docs); + if (algo == PlanningAlgo::Estimate) { + time_ms_esti = res.time_ms; + } + if (algo == PlanningAlgo::CostForceStrict) { + double speedup = time_ms_esti / res.time_ms; + if (speedup > max_speedup) { + max_speedup = speedup; + } + if (speedup < min_speedup) { + min_speedup = speedup; + } + std::cout << "speedup (esti/forc)=" << std::setprecision(4) << speedup << std::endl; + } + } + } + } + std::cout << "max_speedup=" << max_speedup << ", min_speedup=" << min_speedup << std::endl << std::endl; +} + +void +run_and_benchmark(const BlueprintFactorySetup& a, const BlueprintFactorySetup& b, size_t num_docs) +{ + std::cout << "AND[A={" << a.to_string() << "},B={" << b.to_string() << "}]" << std::endl; + run_intermediate_blueprint_benchmark<AndBlueprintFactory>(a, b, num_docs); +} + +//------------------------------------------------------------------------------------- + +std::vector<double> +gen_ratios(double middle, double range_multiplier, size_t num_samples) +{ + double lower = middle / range_multiplier; + double upper = middle * range_multiplier; + // Solve the following equation: + // lower * (factor ^ (num_samples - 1)) = upper; + double factor = std::pow(upper / lower, 1.0 / (num_samples - 1)); + std::vector<double> res; + double ratio = lower; + for (size_t i = 0; i < num_samples; ++i) { + res.push_back(ratio); + ratio *= factor; + if (ratio > 1.0) { + if (res.size() < num_samples) { + res.push_back(1.0); + } + break; + } + } + return res; +} + FieldConfig -make_attr_config(BasicType basic_type, CollectionType col_type, bool fast_search) +make_attr_config(BasicType basic_type, CollectionType col_type, bool fast_search, bool rank_filter = false) { Config cfg(basic_type, col_type); cfg.setFastSearch(fast_search); + cfg.setIsFilter(rank_filter); return FieldConfig(cfg); } @@ -677,6 +836,7 @@ const std::vector<double> base_hit_ratios = {0.0001, 0.001, 0.01, 0.1, 0.5, 1.0} const std::vector<double> filter_hit_ratios = {0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2, 0.5, 1.0}; const auto int32 = make_attr_config(BasicType::INT32, CollectionType::SINGLE, false); const auto int32_fs = make_attr_config(BasicType::INT32, CollectionType::SINGLE, true); +const auto int32_fs_rf = make_attr_config(BasicType::INT32, CollectionType::SINGLE, true, true); const auto int32_array = make_attr_config(BasicType::INT32, CollectionType::ARRAY, false); const auto int32_array_fs = make_attr_config(BasicType::INT32, CollectionType::ARRAY, true); const auto int32_wset = make_attr_config(BasicType::INT32, CollectionType::WSET, false); @@ -694,7 +854,6 @@ TEST(IteratorBenchmark, analyze_term_search_in_disk_index) { BenchmarkSetup setup(num_docs, {str_index}, {QueryOperator::Term}, {true, false}, base_hit_ratios); setup.filter_hit_ratios = filter_hit_ratios; - setup.filter_crossover_factor = 1.0; run_benchmarks(setup, global_summary); } @@ -726,12 +885,21 @@ TEST(IteratorBenchmark, analyze_term_search_in_fast_search_attributes) run_benchmarks(setup, global_summary); } -TEST(IteratorBenchmark, analyze_complex_leaf_operators) +TEST(IteratorBenchmark, analyze_IN_non_strict) +{ + for (auto in_hit_ratio : {0.01, 0.1, 0.5}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {false}, {in_hit_ratio}, {2, 5, 9, 10, 100, 1000, 10000}); + setup.filter_hit_ratios = gen_ratios(in_hit_ratio, 10.0, 13); + setup.disjunct_children = true; + run_benchmarks(setup); + } +} + +TEST(IteratorBenchmark, analyze_IN_strict) { - std::vector<FieldConfig> field_cfgs = {int32_array_fs}; - std::vector<QueryOperator> query_ops = {QueryOperator::In, QueryOperator::DotProduct}; const std::vector<double> hit_ratios = {0.001, 0.01, 0.1, 0.2, 0.4, 0.6, 0.8}; - BenchmarkSetup setup(num_docs, field_cfgs, query_ops, {true, false}, hit_ratios, {1, 2, 10, 100}); + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::In}, {true}, hit_ratios, {2, 5, 9, 10, 100, 1000, 10000}); + setup.disjunct_children = true; run_benchmarks(setup); } @@ -744,40 +912,100 @@ TEST(IteratorBenchmark, analyze_weak_and_operators) run_benchmarks(setup); } -TEST(IteratorBenchmark, term_benchmark) +TEST(IteratorBenchmark, or_vs_filter_crossover) { - BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Term}, {true, false}, base_hit_ratios); - run_benchmarks(setup); + auto fixed_or = make_blueprint_factory(int32_array_fs, QueryOperator::Or, num_docs, 0, 0.1, 100, false); + auto variable_term = [](double rate) { + return make_blueprint_factory(int32_array_fs, QueryOperator::Term, num_docs, 0, rate, 1, false); + }; + analyze_crossover(*fixed_or, variable_term, num_docs + 1, false, 0.0001); +} + +TEST(IteratorBenchmark, or_vs_filter_crossover_with_allow_force_strict) +{ + auto fixed_or = make_blueprint_factory(int32_array_fs, QueryOperator::Or, num_docs, 0, 0.1, 100, false); + auto variable_term = [](double rate) { + return make_blueprint_factory(int32_array_fs, QueryOperator::Term, num_docs, 0, rate, 1, false); + }; + analyze_crossover(*fixed_or, variable_term, num_docs + 1, true, 0.0001); } -TEST(IteratorBenchmark, and_benchmark) +TEST(IteratorBenchmark, analyze_AND_filter_vs_IN) { - BenchmarkSetup setup(num_docs, {int32_array_fs}, {QueryOperator::And}, {true, false}, base_hit_ratios, {1, 2, 4, 8}); + for (auto in_filter_ratio : {0.01, 0.1, 0.5}) { + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(in_filter_ratio, 10.0, 13)}, + {int32_fs, QueryOperator::In, {in_filter_ratio}, children, false}, + num_docs); + } + } +} + +TEST(IteratorBenchmark, analyze_AND_filter_vs_OR) +{ + for (auto or_filter_ratio : {0.01, 0.1, 0.5}) { + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(or_filter_ratio, 10, 13)}, + {int32_fs, QueryOperator::Or, {or_filter_ratio}, children, false}, + num_docs); + } + } +} + +TEST(IteratorBenchmark, analyze_AND_filter_vs_IN_array) +{ + for (uint32_t children: {2, 10, 100, 1000}) { + run_and_benchmark({int32_fs, QueryOperator::Term, gen_ratios(0.1, 10.0, 13)}, + {int32_array_fs, QueryOperator::In, {0.1}, children, false}, + num_docs); + } +} + +TEST(IteratorBenchmark, analyze_AND_bitvector_vs_IN) +{ + for (uint32_t children: {10, 100, 1000, 10000}) { + run_and_benchmark({int32_fs, QueryOperator::In, {0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60}, children, true}, + {int32_fs_rf, QueryOperator::Term, {1.0}, 1, true}, // this setup returns a bitvector matching all documents. + num_docs); + } +} + +TEST(IteratorBenchmark, analyze_OR_non_strict_fs) +{ + for (auto or_hit_ratio : {0.01, 0.1, 0.5}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {false}, {or_hit_ratio}, + {2, 4, 6, 8, 10, 100, 1000}); + setup.filter_hit_ratios = gen_ratios(or_hit_ratio, 10.0, 13); + run_benchmarks(setup); + } +} + +TEST(IteratorBenchmark, analyze_OR_non_strict_non_fs) +{ + BenchmarkSetup setup(num_docs, {int32}, {QueryOperator::Or}, {false}, {0.1}, {2, 4, 6, 8, 10}); + setup.filter_hit_ratios = gen_ratios(0.1, 10.0, 13); run_benchmarks(setup); } -TEST(IteratorBenchmark, or_benchmark) +TEST(IteratorBenchmark, analyze_OR_strict) { - BenchmarkSetup setup(num_docs, {int32_array_fs}, {QueryOperator::Or}, {true, false}, base_hit_ratios, {1, 10, 100, 1000}); + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Or}, {true}, {0.01, 0.1, 0.5}, {2, 4, 6, 8, 10, 100, 1000}); run_benchmarks(setup); } -TEST(IteratorBenchmark, or_vs_filter_crossover) +TEST(IteratorBenchmark, analyze_btree_iterator_non_strict) { - auto fixed_or = make_blueprint_factory(int32_array_fs, QueryOperator::Or, num_docs, 0, 0.1, 100); - auto variable_term = [](double rate) { - return make_blueprint_factory(int32_array_fs, QueryOperator::Term, num_docs, 0, rate, 1); - }; - analyze_crossover(*fixed_or, variable_term, num_docs + 1, false, 0.0001); + for (auto term_ratio : {0.01, 0.1, 0.5, 1.0}) { + BenchmarkSetup setup(num_docs, {int32_fs}, {QueryOperator::Term}, {false}, {term_ratio}, {1}); + setup.filter_hit_ratios = gen_ratios(term_ratio, 10.0, 15); + run_benchmarks(setup); + } } -TEST(IteratorBenchmark, or_vs_filter_crossover_with_allow_force_strict) +TEST(IteratorBenchmark, analyze_btree_vs_bitvector_iterators_strict) { - auto fixed_or = make_blueprint_factory(int32_array_fs, QueryOperator::Or, num_docs, 0, 0.1, 100); - auto variable_term = [](double rate) { - return make_blueprint_factory(int32_array_fs, QueryOperator::Term, num_docs, 0, rate, 1); - }; - analyze_crossover(*fixed_or, variable_term, num_docs + 1, true, 0.0001); + BenchmarkSetup setup(num_docs, {int32_fs, int32_fs_rf}, {QueryOperator::Term}, {true}, {0.1, 0.2, 0.4, 0.5, 0.6, 0.8, 1.0}, {1}); + run_benchmarks(setup); } int main(int argc, char **argv) { diff --git a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp index b7702398857..4ffc1fe366e 100644 --- a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp +++ b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp @@ -44,12 +44,16 @@ public: double calc_distance(uint32_t docid, const vespalib::string& query_tensor) { auto qt = make_tensor(query_tensor); auto calc = DistanceCalculator::make_with_validation(*attr, *qt); - return calc->calc_with_limit(docid, std::numeric_limits<double>::max()); + return calc->has_single_subspace() + ? calc->calc_with_limit<true>(docid, std::numeric_limits<double>::max()) + : calc->calc_with_limit<false>(docid, std::numeric_limits<double>::max()); } double calc_rawscore(uint32_t docid, const vespalib::string& query_tensor) { auto qt = make_tensor(query_tensor); auto calc = DistanceCalculator::make_with_validation(*attr, *qt); - return calc->calc_raw_score(docid); + return calc->has_single_subspace() + ? calc->calc_raw_score<true>(docid) + : calc->calc_raw_score<false>(docid); } OptSubspace calc_closest_subspace(uint32_t docid, const vespalib::string& query_tensor) { auto qt = make_tensor(query_tensor); diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 391e2d91d08..eeae12e1695 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -20,6 +20,16 @@ using search::attribute::DistanceMetric; template <typename T> TypedCells t(const std::vector<T> &v) { return TypedCells(v); } +template<typename T> +struct EmptyCells { + explicit EmptyCells(size_t elems) : _zero(elems, 0), cells(_zero) { cells.size = 0; } + std::vector<T> _zero; + TypedCells cells; +}; + +template <typename T> +EmptyCells<T> e(size_t elems) { return EmptyCells<T>(elems); } + void verify_geo_miles(const std::vector<double> &p1, const std::vector<double> &p2, double exp_miles) @@ -49,6 +59,15 @@ void verify_geo_miles(const std::vector<double> &p1, } } +template<typename T> +void verifyInvalidQueryVector(DistanceFunctionFactory & dff, double expected_distance_to_origo) { + std::vector<T> origo = {0,0,0}; + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<double>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<float>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<Int8Float>(origo.size()).cells)); + EXPECT_FLOAT_EQ(expected_distance_to_origo, dff.for_query_vector(t(origo))->calc(e<vespalib::BFloat16>(origo.size()).cells)); +} + double computeEuclideanChecked(TypedCells a, TypedCells b) { static EuclideanDistanceFunctionFactory<Int8Float> i8f_dff; static EuclideanDistanceFunctionFactory<float> flt_dff; @@ -92,6 +111,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) EXPECT_EQ(d12, 2.0); EuclideanDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); auto euclid = dff.for_query_vector(t(p0)); EXPECT_DOUBLE_EQ(euclid->to_rawscore(d12), 1.0/(1.0 + sqrt(2.0))); double threshold = euclid->convert_threshold(8.0); @@ -128,10 +148,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) EXPECT_EQ(computeEuclideanChecked(t(p6), t(p6)), 0.0); // smoke test for bfloat16: - std::vector<vespalib::BFloat16> bf16v; - bf16v.emplace_back(1.0); - bf16v.emplace_back(1.0); - bf16v.emplace_back(1.0); + std::vector<vespalib::BFloat16> bf16v{1.0, 1.0, 1.0}; EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p0)), 3.0); EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p1)), 2.0); EXPECT_EQ(computeEuclideanChecked(t(bf16v), t(p2)), 2.0); @@ -188,6 +205,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) AngularDistanceFunctionFactory<double> dff; auto angular = dff.for_query_vector(t(p0)); + verifyInvalidQueryVector<double>(dff, 1.0); constexpr double pi = 3.14159265358979323846; double a12 = computeAngularChecked(t(p1), t(p2)); double a13 = computeAngularChecked(t(p1), t(p3)); @@ -315,6 +333,7 @@ TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) std::vector<double> p8{3.0, 0.0, 0.0}; PrenormalizedAngularDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 1.0); auto pnad = dff.for_query_vector(t(p0)); double i12 = computePrenormalizedAngularChecked(t(p1), t(p2)); @@ -360,7 +379,8 @@ TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) TEST(DistanceFunctionsTest, hamming_gives_expected_score) { - static HammingDistanceFunctionFactory<double> dff; + HammingDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); std::vector<std::vector<double>> points{{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, @@ -376,6 +396,7 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) EXPECT_EQ(h0, 0.0); EXPECT_EQ(dist_fun->to_rawscore(h0), 1.0); } + double d12 = dff.for_query_vector(t(points[1]))->calc(t(points[2])); EXPECT_EQ(d12, 3.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d12), 1.0/(1.0 + 3.0)); @@ -579,6 +600,9 @@ TEST(DistanceFunctionsTest, transformed_mips_basic_scores) std::vector<double> p4{0.5, 0.5, sq_root_half}; std::vector<double> p5{0.0,-1.0, 0.0}; + MipsDistanceFunctionFactory<double> dff; + verifyInvalidQueryVector<double>(dff, 0.0); + double i12 = computeTransformedMipsChecked(t(p1), t(p2)); double i13 = computeTransformedMipsChecked(t(p1), t(p3)); double i23 = computeTransformedMipsChecked(t(p2), t(p3)); diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index c01fc33767a..31b147c0b5c 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -20,6 +20,7 @@ #include <vespa/vespalib/util/fake_doom.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <type_traits> #include <vector> @@ -46,13 +47,19 @@ class MyDocVectorAccess : public DocVectorAccess { private: using Vector = std::vector<FloatType>; using ArrayRef = vespalib::ConstArrayRef<FloatType>; - std::vector<Vector> _vectors; - SubspaceType _subspace_type; + mutable std::vector<Vector> _vectors; + SubspaceType _subspace_type; + mutable uint32_t _get_vector_count; + mutable uint32_t _schedule_clear_tensor; + mutable uint32_t _cleared_tensor_docid; public: MyDocVectorAccess() : _vectors(), - _subspace_type(ValueType::make_type(get_cell_type<FloatType>(), {{"dims", 2}})) + _subspace_type(ValueType::make_type(get_cell_type<FloatType>(), {{"dims", 2}})), + _get_vector_count(0), + _schedule_clear_tensor(0), + _cleared_tensor_docid(0) { } MyDocVectorAccess& set(uint32_t docid, const Vector& vec) { @@ -62,19 +69,101 @@ public: _vectors[docid] = vec; return *this; } - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override { - return get_vectors(docid).cells(subspace); + void clear(uint32_t docid) const { + if (docid < _vectors.size()) { + _vectors[docid].clear(); + } + } + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override { + ++_get_vector_count; + if (_schedule_clear_tensor != 0) { + if (--_schedule_clear_tensor == 0) { + // Simulate race where writer thread has cleared a tensor. + clear(docid); + _cleared_tensor_docid = docid; + } + } + auto bundle = get_vectors(docid); + if (subspace < bundle.subspaces()) { + return bundle.cells(subspace); + } + return { nullptr, _subspace_type.cell_type(), 0 }; } - VectorBundle get_vectors(uint32_t docid) const override { + VectorBundle get_vectors(uint32_t docid) const noexcept override { ArrayRef ref(_vectors[docid]); assert((ref.size() % _subspace_type.size()) == 0); uint32_t subspaces = ref.size() / _subspace_type.size(); - return VectorBundle(ref.data(), subspaces, _subspace_type); + return {ref.data(), subspaces, _subspace_type}; } void clear() { _vectors.clear(); } + + uint32_t get_vector_count() const noexcept { return _get_vector_count; } + void clear_cleared_tensor_docid() { _cleared_tensor_docid = 0; } + uint32_t get_cleared_tensor_docid() const noexcept { return _cleared_tensor_docid; } + void set_schedule_clear_tensor(uint32_t v) { _schedule_clear_tensor = v; } +}; + +class MyBoundDistanceFunction : public BoundDistanceFunction { + std::unique_ptr<BoundDistanceFunction> _real; + +public: + MyBoundDistanceFunction(std::unique_ptr<BoundDistanceFunction> real) + : _real(std::move(real)) + { + } + + ~MyBoundDistanceFunction() override; + + double convert_threshold(double threshold) const noexcept override { + return _real->convert_threshold(threshold); + } + double to_rawscore(double distance) const noexcept override { + return _real->to_rawscore(distance); + } + double to_distance(double rawscore) const noexcept override { + return _real->to_distance(rawscore); + } + + double min_rawscore() const noexcept override { return _real->min_rawscore(); } + + double calc(TypedCells rhs) const noexcept override { + EXPECT_TRUE(rhs.valid()); + return _real->calc(rhs); + } + + double calc_with_limit(TypedCells rhs, double limit) const noexcept override { + EXPECT_TRUE(rhs.valid()); + return _real->calc_with_limit(rhs, limit); + } +}; + +MyBoundDistanceFunction::~MyBoundDistanceFunction() = default; + +class MyDistanceFunctionFactory : public DistanceFunctionFactory +{ + std::unique_ptr<DistanceFunctionFactory> _real; +public: + MyDistanceFunctionFactory(std::unique_ptr<DistanceFunctionFactory> real) + : _real(std::move(real)) + { + } + + ~MyDistanceFunctionFactory() override; + + std::unique_ptr<BoundDistanceFunction> for_query_vector(TypedCells lhs) override { + EXPECT_TRUE(lhs.valid()); + return std::make_unique<MyBoundDistanceFunction>(_real->for_query_vector(lhs)); + } + + std::unique_ptr<BoundDistanceFunction> for_insertion_vector(TypedCells lhs) override { + EXPECT_TRUE(lhs.valid()); + return std::make_unique<MyBoundDistanceFunction>(_real->for_insertion_vector(lhs)); + } }; +MyDistanceFunctionFactory::~MyDistanceFunctionFactory() = default; + struct LevelGenerator : public RandomLevelGenerator { uint32_t level; LevelGenerator() : level(0) {} @@ -106,14 +195,18 @@ public: .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5}); } - ~HnswIndexTest() override {} + ~HnswIndexTest() override; - auto dff() { + auto dff_real() { return search::tensor::make_distance_function_factory( search::attribute::DistanceMetric::Euclidean, vespalib::eval::CellType::FLOAT); } + auto dff() { + return std::make_unique<MyDistanceFunctionFactory>(dff_real()); + } + void init(bool heuristic_select_neighbors) { auto generator = std::make_unique<LevelGenerator>(); level_generator = generator.get(); @@ -277,9 +370,23 @@ public: return index->get_active_nodes(); } + /* + * Simulate race where writer has cleared a tensor while read thread still + * use old graph. + */ + void writer_clears_tensor(uint32_t docid) { vectors.clear(docid); } + + uint32_t get_vector_count() const noexcept { return vectors.get_vector_count(); } + void clear_cleared_tensor_docid() { vectors.clear_cleared_tensor_docid(); } + uint32_t get_cleared_tensor_docid() const noexcept { return vectors.get_cleared_tensor_docid(); } + void set_schedule_clear_tensor(uint32_t v) { vectors.set_schedule_clear_tensor(v); } + static constexpr bool is_single = std::is_same_v<IndexType, HnswIndex<HnswIndexType::SINGLE>>; }; +template <typename IndexType> +HnswIndexTest<IndexType>::~HnswIndexTest() = default; + using HnswIndexTestTypes = ::testing::Types<HnswIndex<HnswIndexType::SINGLE>, HnswIndex<HnswIndexType::MULTI>>; TYPED_TEST_SUITE(HnswIndexTest, HnswIndexTestTypes); @@ -529,11 +636,9 @@ TYPED_TEST(HnswIndexTest, manual_insert) this->init(false); EXPECT_EQ(0, this->get_active_nodes()); - std::vector<uint32_t> nbl; - HnswTestNode empty{nbl}; - this->index->set_node(1, empty); + this->index->set_node(1, std::vector<uint32_t>()); EXPECT_EQ(1, this->get_active_nodes()); - this->index->set_node(2, empty); + this->index->set_node(2, std::vector<uint32_t>()); EXPECT_EQ(2, this->get_active_nodes()); HnswTestNode three{{1,2}}; @@ -545,7 +650,7 @@ TYPED_TEST(HnswIndexTest, manual_insert) this->expect_entry_point(1, 0); - HnswTestNode twolevels{{{1},nbl}}; + HnswTestNode twolevels{{{1},std::vector<uint32_t>()}}; this->index->set_node(4, twolevels); this->expect_entry_point(4, 1); @@ -623,11 +728,9 @@ TYPED_TEST(HnswIndexTest, memory_is_put_on_hold_while_read_guard_is_held) TYPED_TEST(HnswIndexTest, shrink_called_simple) { this->init(false); - std::vector<uint32_t> nbl; - HnswTestNode empty{nbl}; + HnswTestNode empty{std::vector<uint32_t>()}; this->index->set_node(1, empty); - nbl.push_back(1); - HnswTestNode nb1{nbl}; + HnswTestNode nb1{std::vector<uint32_t>(1, 1)}; this->index->set_node(2, nb1); this->index->set_node(3, nb1); this->index->set_node(4, nb1); @@ -663,11 +766,9 @@ TYPED_TEST(HnswIndexTest, shrink_called_simple) TYPED_TEST(HnswIndexTest, shrink_called_heuristic) { this->init(true); - std::vector<uint32_t> nbl; - HnswTestNode empty{nbl}; + HnswTestNode empty{std::vector<uint32_t>()}; this->index->set_node(1, empty); - nbl.push_back(1); - HnswTestNode nb1{nbl}; + HnswTestNode nb1{std::vector<uint32_t>(1, 1)}; this->index->set_node(2, nb1); this->index->set_node(3, nb1); this->index->set_node(4, nb1); @@ -824,6 +925,14 @@ TYPED_TEST(HnswIndexTest, hnsw_graph_can_be_saved_and_loaded) this->check_savetest_index("after load"); } +TYPED_TEST(HnswIndexTest, search_during_remove) +{ + this->init(false); + this->make_savetest_index(); + this->writer_clears_tensor(4); + this->expect_top_3_by_docid("{0, 0}", {0, 0}, {7}); +} + using HnswMultiIndexTest = HnswIndexTest<HnswIndex<HnswIndexType::MULTI>>; namespace { @@ -962,6 +1071,7 @@ public: .set(1, {3, 7}).set(2, {7, 1}).set(3, {11, 7}) .set(7, {6, 5}).set(8, {5, 5}).set(9, {6, 6}); } + ~TwoPhaseTest() override; using UP = std::unique_ptr<PrepareResult>; UP prepare_add(uint32_t docid, uint32_t max_level = 0) { this->level_generator->level = max_level; @@ -973,8 +1083,49 @@ public: this->index->complete_add_document(docid, std::move(up)); this->commit(); } + + uint32_t prepare_insert_during_remove_pass(bool heuristic_select_neighbors, uint32_t schedule_clear_tensor, const vespalib::string& label); + void prepare_insert_during_remove(bool heuristic_select_neighbors); }; +template <typename IndexType> +TwoPhaseTest<IndexType>::~TwoPhaseTest() = default; + +template <typename IndexType> +uint32_t +TwoPhaseTest<IndexType>::prepare_insert_during_remove_pass(bool heuristic_select_neighbors, uint32_t schedule_clear_tensor, const vespalib::string& label) +{ + SCOPED_TRACE(label); + this->init(heuristic_select_neighbors); + this->vectors.clear(); + this->vectors.set(4, {1, 3}).set(2, {7, 1}).set(7, {6, 5}); + this->make_savetest_index(); + auto old_get_vector_count = this->get_vector_count(); + this->set_schedule_clear_tensor(schedule_clear_tensor); + this->clear_cleared_tensor_docid(); + auto prepared = this->prepare_add(2, 1); + auto result = this->get_vector_count() - old_get_vector_count; + auto cleared_tensor_docid = this->get_cleared_tensor_docid(); + if (cleared_tensor_docid != 0) { + this->remove_document(cleared_tensor_docid); + } + this->complete_add(2, std::move(prepared)); + EXPECT_EQ(((cleared_tensor_docid == 0u) ? 3 : 2), this->get_active_nodes()); + return result; +} + +template <typename IndexType> +void +TwoPhaseTest<IndexType>::prepare_insert_during_remove(bool heuristic_select_neighbors) +{ + auto get_vector_counts = prepare_insert_during_remove_pass(heuristic_select_neighbors, 0, "No clear tensor"); + for (uint32_t schedule_clear_tensor = 1; schedule_clear_tensor <= get_vector_counts; ++schedule_clear_tensor) { + vespalib::asciistream os; + os << "Writer thread cleared tensor for get_vector (" << schedule_clear_tensor << " of " << get_vector_counts << ")"; + prepare_insert_during_remove_pass(heuristic_select_neighbors, schedule_clear_tensor, os.str()); + } +} + TYPED_TEST_SUITE(TwoPhaseTest, HnswIndexTestTypes); TYPED_TEST(TwoPhaseTest, two_phase_add) @@ -1019,4 +1170,14 @@ TYPED_TEST(TwoPhaseTest, two_phase_add) this->expect_levels(nodeids[0], {{2}, {4}}); } +TYPED_TEST(TwoPhaseTest, prepare_insert_during_remove_simple_select_neighbors) +{ + this->prepare_insert_during_remove(false); +} + +TYPED_TEST(TwoPhaseTest, prepare_insert_during_remove_heuristic_select_neighbors) +{ + this->prepare_insert_during_remove(true); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index 1feb968fbb4..dce09a87fb8 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -1,13 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <fcntl.h> -#include <cstdio> -#include <unistd.h> -#include <chrono> -#include <cstdlib> -#include <future> -#include <vector> - #include <vespa/eval/eval/typed_cells.h> #include <vespa/eval/eval/value_type.h> #include <vespa/searchlib/common/bitvector.h> @@ -25,6 +17,9 @@ #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <fcntl.h> +#include <unistd.h> +#include <future> #include <vespa/log/log.h> LOG_SETUP("stress_hnsw_mt"); @@ -119,17 +114,17 @@ public: memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector)); return *this; } - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override { + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override { assert(docid < NUM_POSSIBLE_DOCS); (void) subspace; ConstVectorRef ref(_vectors[docid]); return vespalib::eval::TypedCells(ref); } - VectorBundle get_vectors(uint32_t docid) const override { + VectorBundle get_vectors(uint32_t docid) const noexcept override { assert(docid < NUM_POSSIBLE_DOCS); ConstVectorRef ref(_vectors[docid]); assert(subspace_type.size() == ref.size()); - return VectorBundle(ref.data(), 1, subspace_type); + return {ref.data(), 1, subspace_type}; } }; @@ -257,7 +252,7 @@ public: loaded_vectors.load(); } - ~Stressor() {} + ~Stressor() override; auto dff() { return search::tensor::make_distance_function_factory( @@ -352,6 +347,9 @@ public: } }; +template <typename IndexType> +Stressor<IndexType>::~Stressor() = default; + using StressorTypes = ::testing::Types<HnswIndex<HnswIndexType::SINGLE>>; TYPED_TEST_SUITE(Stressor, StressorTypes); diff --git a/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp index 43dc5333485..9f6cd8cc114 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_operations/tensor_buffer_operations_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/streamed/streamed_value_builder_factory.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/gtest/gtest.h> using search::tensor::TensorBufferOperations; diff --git a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp index bb0e2e1a3dd..07d0628bb16 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_store/tensor_buffer_store_test.cpp @@ -8,6 +8,9 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/objects/nbostream.h> + + using search::tensor::TensorBufferStore; using vespalib::datastore::EntryRef; diff --git a/searchlib/src/vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h b/searchlib/src/vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h new file mode 100644 index 00000000000..33031069d8e --- /dev/null +++ b/searchlib/src/vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h @@ -0,0 +1,33 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "hit_estimate.h" +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> + +namespace search::attribute { + +/** + * Adapter used when calculating FlowStats based on HitEstimate per term. + */ +struct HitEstimateFlowStatsAdapter { + uint32_t docid_limit; + uint32_t num_indirections; + explicit HitEstimateFlowStatsAdapter(uint32_t docid_limit_in, uint32_t num_indirections_in) noexcept + : docid_limit(docid_limit_in), num_indirections(num_indirections_in) {} + double abs_to_rel_est(const HitEstimate& est) const noexcept { + return queryeval::Blueprint::abs_to_rel_est(est.est_hits(), docid_limit); + } + double estimate(const HitEstimate& est) const noexcept { + return est.is_unknown() ? 0.5 : abs_to_rel_est(est); + } + double cost(const HitEstimate& est) const noexcept { + return est.is_unknown() ? queryeval::flow::lookup_cost(num_indirections) : queryeval::flow::btree_cost(abs_to_rel_est(est)); + } + double strict_cost(const HitEstimate &est) const noexcept { + return est.is_unknown() ? queryeval::flow::lookup_strict_cost(num_indirections) : queryeval::flow::btree_strict_cost(abs_to_rel_est(est)); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index aebfda8fffd..5b17b491a20 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -5,6 +5,7 @@ #include "attribute_object_visitor.h" #include "attribute_weighted_set_blueprint.h" #include "direct_multi_term_blueprint.h" +#include "direct_posting_store_flow_stats_adapter.h" #include "i_docid_posting_store.h" #include "i_docid_with_weight_posting_store.h" #include "in_term_search.h" @@ -12,6 +13,7 @@ #include "predicate_attribute.h" #include <vespa/eval/eval/value.h> #include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h> #include <vespa/searchlib/common/location.h> #include <vespa/searchlib/common/locationiterators.h> #include <vespa/searchlib/query/query_term_decoder.h> @@ -24,23 +26,23 @@ #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/field_spec.hpp> #include <vespa/searchlib/queryeval/filter_wrapper.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/queryeval/get_weight_from_node.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> +#include <vespa/searchlib/queryeval/irequestcontext.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/queryeval/orlikesearch.h> -#include <vespa/searchlib/queryeval/flow_tuning.h> #include <vespa/searchlib/queryeval/predicate_blueprint.h> #include <vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h> #include <vespa/searchlib/queryeval/wand/parallel_weak_and_search.h> #include <vespa/searchlib/queryeval/weighted_set_term_blueprint.h> #include <vespa/searchlib/queryeval/weighted_set_term_search.h> -#include <vespa/searchlib/queryeval/irequestcontext.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/vespalib/util/regexp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/util/regexp.h> +#include <vespa/vespalib/util/stringfmt.h> #include <charconv> #include <vespa/log/log.h> @@ -92,6 +94,7 @@ using search::queryeval::StrictHeapOrSearch; using search::queryeval::WeightedSetTermBlueprint; using search::queryeval::flow::btree_cost; using search::queryeval::flow::btree_strict_cost; +using search::queryeval::flow::get_num_indirections; using search::queryeval::flow::lookup_cost; using search::queryeval::flow::lookup_strict_cost; using search::tensor::DenseTensorAttribute; @@ -122,19 +125,6 @@ private: }; //----------------------------------------------------------------------------- -size_t -get_num_indirections(const BasicType& basic_type, const CollectionType& col_type) -{ - size_t res = 0; - if (basic_type == BasicType::STRING) { - res += 1; - } - if (col_type != CollectionType::SINGLE) { - res += 1; - } - return res; -} - /** * Blueprint for creating regular, stack-based attribute iterators. **/ @@ -166,7 +156,7 @@ public: return {0.5, lookup_cost(indirections), lookup_strict_cost(indirections)}; } else { double rel_est = abs_to_rel_est(_hit_estimate.est_hits(), docid_limit); - return {rel_est, btree_cost(), btree_strict_cost(rel_est)}; + return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; } } @@ -292,20 +282,11 @@ public: } queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { using OrFlow = search::queryeval::OrFlow; - struct MyAdapter { - uint32_t docid_limit; - explicit MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} - double estimate(const AttrHitEstimate &est) const noexcept { - return est.is_unknown() ? 0.5 : abs_to_rel_est(est.est_hits(), docid_limit); - } - double cost(const AttrHitEstimate &) const noexcept { return 1.0; } - double strict_cost(const AttrHitEstimate &est) const noexcept { - return est.is_unknown() ? 1.0 : abs_to_rel_est(est.est_hits(), docid_limit); - } - }; - double est = OrFlow::estimate_of(MyAdapter(docid_limit), _estimates); - return {est, OrFlow::cost_of(MyAdapter(docid_limit), _estimates, false), - OrFlow::cost_of(MyAdapter(docid_limit), _estimates, true) + queryeval::flow::heap_cost(est, _estimates.size())}; + using MyAdapter = attribute::HitEstimateFlowStatsAdapter; + size_t indirections = queryeval::flow::get_num_indirections(_attribute.getBasicType(), _attribute.getCollectionType()); + double est = OrFlow::estimate_of(MyAdapter(docid_limit, indirections), _estimates); + return {est, OrFlow::cost_of(MyAdapter(docid_limit, indirections), _estimates, false), + OrFlow::cost_of(MyAdapter(docid_limit, indirections), _estimates, true) + queryeval::flow::heap_cost(est, _estimates.size())}; } SearchIterator::UP @@ -513,20 +494,7 @@ public: queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { using OrFlow = search::queryeval::OrFlow; - struct MyAdapter { - uint32_t docid_limit; - explicit MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} - double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { - return abs_to_rel_est(term.posting_size, docid_limit); - } - double cost(const IDirectPostingStore::LookupResult &) const noexcept { - return btree_cost(); - } - double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { - double rel_est = abs_to_rel_est(term.posting_size, docid_limit); - return btree_strict_cost(rel_est); - } - }; + using MyAdapter = attribute::DirectPostingStoreFlowStatsAdapter; double child_est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); double my_est = abs_to_rel_est(_scores.getScoresToTrack(), docid_limit); double est = (child_est + my_est) / 2.0; diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 2ad67c85429..928023c0f94 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -2,20 +2,19 @@ #include "attribute_weighted_set_blueprint.h" #include "multi_term_hash_filter.hpp" +#include <vespa/searchcommon/attribute/hit_estimate_flow_stats_adapter.h> #include <vespa/searchcommon/attribute/i_search_context.h> -#include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/query/query_term_ucs4.h> #include <vespa/searchlib/queryeval/filter_wrapper.h> -#include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/searchlib/queryeval/flow.h> #include <vespa/searchlib/queryeval/flow_tuning.h> +#include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/searchlib/queryeval/weighted_set_term_search.h> #include <vespa/vespalib/objects/visit.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/stringfmt.h> - namespace search { namespace { @@ -130,20 +129,11 @@ AttributeWeightedSetBlueprint::sort(queryeval::InFlow in_flow) queryeval::FlowStats AttributeWeightedSetBlueprint::calculate_flow_stats(uint32_t docid_limit) const { - struct MyAdapter { - uint32_t docid_limit; - MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} - double estimate(const AttrHitEstimate &est) const noexcept { - return est.is_unknown() ? 0.5 : abs_to_rel_est(est.est_hits(), docid_limit); - } - double cost(const AttrHitEstimate &) const noexcept { return 1.0; } - double strict_cost(const AttrHitEstimate &est) const noexcept { - return est.is_unknown() ? 1.0 : abs_to_rel_est(est.est_hits(), docid_limit); - } - }; - double est = OrFlow::estimate_of(MyAdapter(docid_limit), _estimates); - return {est, OrFlow::cost_of(MyAdapter(docid_limit), _estimates, false), - OrFlow::cost_of(MyAdapter(docid_limit), _estimates, true) + queryeval::flow::heap_cost(est, _estimates.size())}; + using MyAdapter = attribute::HitEstimateFlowStatsAdapter; + size_t num_indirections = queryeval::flow::get_num_indirections(_attr.getBasicType(), _attr.getCollectionType()); + double est = OrFlow::estimate_of(MyAdapter(docid_limit, num_indirections), _estimates); + return {est, OrFlow::cost_of(MyAdapter(docid_limit, num_indirections), _estimates, false), + OrFlow::cost_of(MyAdapter(docid_limit, num_indirections), _estimates, true) + queryeval::flow::heap_cost(est, _estimates.size())}; } queryeval::SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp index 5f3ab9cd3d8..3ae2bdd3598 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp @@ -44,8 +44,12 @@ extract_suffix(std::string_view target, uint32_t prefix_size) } -DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, LevenshteinDfa::DfaType dfa_type) - : _dfa(vespalib::fuzzy::LevenshteinDfa::build(extract_suffix(target, prefix_size), max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), +DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, + bool cased, bool prefix_match, LevenshteinDfa::DfaType dfa_type) + : _dfa(vespalib::fuzzy::LevenshteinDfa::build(extract_suffix(target, prefix_size), max_edits, + (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), + dfa_type, // TODO reorder args + (prefix_match ? LevenshteinDfa::Matching::Prefix : LevenshteinDfa::Matching::FullString))), _successor(), _prefix(extract_prefix(target, prefix_size, cased)), _prefix_size(prefix_size), @@ -54,8 +58,8 @@ DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uin _successor = _prefix; } -DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased) - : DfaFuzzyMatcher(target, max_edits, prefix_size, cased, LevenshteinDfa::DfaType::Table) +DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, bool prefix_match) + : DfaFuzzyMatcher(target, max_edits, prefix_size, cased, prefix_match, LevenshteinDfa::DfaType::Table) { } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index 653af602c0d..ee57e9e6583 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -26,8 +26,10 @@ private: const char* skip_prefix(const char* word) const; public: - DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); - DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased); // Defaults to table-based DFA + DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, bool prefix_match, + vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); + // Defaults to table-based DFA: + DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, bool prefix_match); ~DfaFuzzyMatcher(); [[nodiscard]] static constexpr bool supports_max_edits(uint8_t edits) noexcept { diff --git a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h index 1759ca71432..51321a56885 100644 --- a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.h @@ -76,26 +76,7 @@ public: resolve_strict(in_flow); } - queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { - using OrFlow = search::queryeval::OrFlow; - struct MyAdapter { - uint32_t docid_limit; - MyAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} - double estimate(const IDirectPostingStore::LookupResult &term) const noexcept { - return abs_to_rel_est(term.posting_size, docid_limit); - } - double cost(const IDirectPostingStore::LookupResult &) const noexcept { - return search::queryeval::flow::btree_cost(); - } - double strict_cost(const IDirectPostingStore::LookupResult &term) const noexcept { - double rel_est = abs_to_rel_est(term.posting_size, docid_limit); - return search::queryeval::flow::btree_strict_cost(rel_est); - } - }; - double est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); - return {est, OrFlow::cost_of(MyAdapter(docid_limit), _terms, false), - OrFlow::cost_of(MyAdapter(docid_limit), _terms, true) + queryeval::flow::heap_cost(est, _terms.size())}; - } + queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override; std::unique_ptr<queryeval::SearchIterator> createLeafSearch(const fef::TermFieldMatchDataArray &tfmda) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp index 817eab3e070..392a4da685c 100644 --- a/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp +++ b/searchlib/src/vespa/searchlib/attribute/direct_multi_term_blueprint.hpp @@ -3,6 +3,7 @@ #pragma once #include "direct_multi_term_blueprint.h" +#include "direct_posting_store_flow_stats_adapter.h" #include "multi_term_or_filter_search.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/queryeval/emptysearch.h> @@ -181,4 +182,21 @@ DirectMultiTermBlueprint<PostingStoreType, SearchType>::createFilterSearch(Filte return wrapper; } +template <typename PostingStoreType, typename SearchType> +queryeval::FlowStats +DirectMultiTermBlueprint<PostingStoreType, SearchType>::calculate_flow_stats(uint32_t docid_limit) const +{ + using OrFlow = search::queryeval::OrFlow; + using MyAdapter = DirectPostingStoreFlowStatsAdapter; + double est = OrFlow::estimate_of(MyAdapter(docid_limit), _terms); + // Iterator benchmarking has shown that non-strict cost is different for attributes + // that support using a reverse hash filter (see use_hash_filter()). + // Program used: searchlib/src/tests/queryeval/iterator_benchmark + // Tests: analyze_and_with_filter_vs_in(), analyze_and_with_filter_vs_in_array() + double non_strict_cost = (SearchType::supports_hash_filter && !_iattr.hasMultiValue()) + ? queryeval::flow::reverse_hash_lookup() + : OrFlow::cost_of(MyAdapter(docid_limit), _terms, false); + return {est, non_strict_cost, OrFlow::cost_of(MyAdapter(docid_limit), _terms, true) + queryeval::flow::heap_cost(est, _terms.size())}; +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/direct_posting_store_flow_stats_adapter.h b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_flow_stats_adapter.h new file mode 100644 index 00000000000..56ac5043148 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/direct_posting_store_flow_stats_adapter.h @@ -0,0 +1,30 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_direct_posting_store.h" +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/flow_tuning.h> + +namespace search::attribute { + +/** + * Adapter used when calculating FlowStats based on IDirectPostingStore::LookupResult per term. + */ +struct DirectPostingStoreFlowStatsAdapter { + uint32_t docid_limit; + DirectPostingStoreFlowStatsAdapter(uint32_t docid_limit_in) noexcept : docid_limit(docid_limit_in) {} + double estimate(const IDirectPostingStore::LookupResult& term) const noexcept { + return queryeval::Blueprint::abs_to_rel_est(term.posting_size, docid_limit); + } + double cost(const IDirectPostingStore::LookupResult& term) const noexcept { + double rel_est = queryeval::Blueprint::abs_to_rel_est(term.posting_size, docid_limit); + return queryeval::flow::btree_cost(rel_est); + } + double strict_cost(const IDirectPostingStore::LookupResult& term) const noexcept { + double rel_est = queryeval::Blueprint::abs_to_rel_est(term.posting_size, docid_limit); + return queryeval::flow::btree_strict_cost(rel_est); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h index 28e7094091f..b0fc835a34b 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute_saver.h @@ -4,6 +4,7 @@ #include "attributesaver.h" #include <vespa/vespalib/stllike/allocator.h> +#include <memory> namespace search::predicate { class ISaver; } diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp index f1a643dc376..c6cb6e4a637 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp +++ b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp @@ -49,17 +49,19 @@ StringSearchHelper::StringSearchHelper(QueryTermUCS4 & term, bool cased, vespali ? vespalib::Regex::from_pattern(term.getTerm(), vespalib::Regex::Options::None) : vespalib::Regex::from_pattern(term.getTerm(), vespalib::Regex::Options::IgnoreCase); } else if (isFuzzy()) { - auto max_edit_dist = term.getFuzzyMaxEditDistance(); + const auto max_edit_dist = term.fuzzy_max_edit_distance(); _fuzzyMatcher = std::make_unique<vespalib::FuzzyMatcher>(term.getTerm(), max_edit_dist, - term.getFuzzyPrefixLength(), - isCased()); + term.fuzzy_prefix_lock_length(), + isCased(), + term.fuzzy_prefix_match()); if ((fuzzy_matching_algorithm != FMA::BruteForce) && (max_edit_dist > 0 && max_edit_dist <= 2)) { _dfa_fuzzy_matcher = std::make_unique<DfaFuzzyMatcher>(term.getTerm(), max_edit_dist, - term.getFuzzyPrefixLength(), + term.fuzzy_prefix_lock_length(), isCased(), + term.fuzzy_prefix_match(), to_dfa_type(fuzzy_matching_algorithm)); } } else if (isCased()) { diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 0f089c60e4b..e5ce886f499 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -6,7 +6,9 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/datastore/aligner.h> #include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/round_up_to_page_size.h> #include <vespa/vespalib/util/size_literals.h> namespace search::bitcompression { @@ -181,6 +183,12 @@ readHeader(vespalib::GenericHeader &header, int64_t fileSize) return headerLen; } +bool +DecodeContext64Base::is_padded_for_memory_map(uint64_t file_bit_size, uint64_t file_size) noexcept +{ + using Aligner = vespalib::datastore::Aligner<64>; + return (Aligner::align(file_bit_size) + 128 <= (vespalib::round_up_to_page_size(file_size) * 8)); +} template <bool bigEndian> void @@ -359,6 +367,24 @@ getParams(PostingListParams ¶ms) const params.clear(); } +template <bool bigEndian> +void +FeatureEncodeContext<bigEndian>::pad_for_memory_map_and_flush() +{ + // Write some pad bits to avoid decompression readahead going past + // memory mapped file during search and into SIGSEGV territory. + + // First pad to 64 bits alignment. + this->smallAlign(64); + writeComprBufferIfNeeded(); + + // Then write 128 more bits. This allows for 64-bit decoding + // with a readbits that always leaves a nonzero preRead + padBits(128); + this->alignDirectIO(); + this->flush(); + writeComprBuffer(); // Also flushes slack +} template <bool bigEndian> void diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 232572a6314..b1e13a9d96b 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -1167,7 +1167,7 @@ public: * Get remaining units in buffer (e.g. _realValE - _valI) */ - int32_t remainingUnits() const override { return _realValE - _valI; } + int64_t remainingUnits() const override { return _realValE - _valI; } /** * Get unit ptr (e.g. _valI) from decode context. @@ -1181,7 +1181,7 @@ public: } uint64_t getBitPos(int bitOffset, uint64_t bufferEndFilePos) const override { - int intOffset = _realValE - _valI; + int64_t intOffset = _realValE - _valI; if (bitOffset == -1) { bitOffset = -64 - _preRead; } @@ -1200,7 +1200,7 @@ public: uint64_t getBitPosV() const override { return getReadOffset(); } - void adjUnitPtr(int newRemainingUnits) override { + void adjUnitPtr(int64_t newRemainingUnits) override { _valI = _realValE - newRemainingUnits; } @@ -1219,7 +1219,7 @@ public: * @param unitCount Number of bytes in buffer * @param moreData Set if there is more data available */ - void setEnd(unsigned int unitCount, bool moreData) { + void setEnd(uint64_t unitCount, bool moreData) { _valE = _realValE = _valI + unitCount; if (moreData) { _valE -= END_BUFFER_SAFETY; @@ -1261,6 +1261,13 @@ public: virtual uint64_t decode_exp_golomb(int k) = 0; void readBytes(uint8_t *buf, size_t len); uint32_t readHeader(vespalib::GenericHeader &header, int64_t fileSize); + + /* + * Check if file is padding at end for decompression readahead. + */ + static bool is_padded_for_memory_map(uint64_t file_bit_size, uint64_t file_size) noexcept; + + static uint64_t file_units(uint64_t file_size) noexcept { return (file_size + sizeof(uint64_t) - 1) / sizeof(uint64_t); } }; @@ -1595,6 +1602,8 @@ public: writeComprBufferIfNeeded(); } + void pad_for_memory_map_and_flush(); + virtual void readHeader(const vespalib::GenericHeader &header, const vespalib::string &prefix); virtual void writeHeader(vespalib::GenericHeader &header, const vespalib::string &prefix) const; virtual const vespalib::string &getIdentifier() const; diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index c0d0b1baa8c..c30b02a0c37 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -72,7 +72,7 @@ queryeval::FlowStats DiskTermBlueprint::calculate_flow_stats(uint32_t docid_limit) const { double rel_est = abs_to_rel_est(_lookupRes->counts._numDocs, docid_limit); - return {rel_est, disk_index_cost(), disk_index_strict_cost(rel_est)}; + return {rel_est, disk_index_cost(rel_est), disk_index_strict_cost(rel_est)}; } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index 9fb966a06cf..fb940026735 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -4,6 +4,7 @@ #include "fusion_output_index.h" #include <vespa/vespalib/util/executor.h> +#include <vespa/vespalib/util/array.h> namespace search { class IFlushToken; diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 387d95bce66..3991807e731 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -51,7 +51,7 @@ using vespalib::getLastErrorString; namespace search::diskindex { struct PageDict4FileSeqRead::DictFileReadContext { - DictFileReadContext(vespalib::stringref id, const vespalib::string & name, const TuneFileSeqRead &tune, bool read_all_upfront); + DictFileReadContext(vespalib::stringref id, const vespalib::string & name, const TuneFileSeqRead &tune, uint32_t mmap_file_size_threshold, bool read_all_upfront); ~DictFileReadContext(); vespalib::FileHeader readHeader(); void readExtendedHeader(); @@ -66,7 +66,7 @@ struct PageDict4FileSeqRead::DictFileReadContext { }; PageDict4FileSeqRead::DictFileReadContext::DictFileReadContext(vespalib::stringref id, const vespalib::string & name, - const TuneFileSeqRead &tune, bool read_all_upfront) + const TuneFileSeqRead &tune, uint32_t mmap_file_size_threshold, bool read_all_upfront) : _id(id), _fileBitSize(0u), _headerLen(0u), @@ -76,26 +76,52 @@ PageDict4FileSeqRead::DictFileReadContext::DictFileReadContext(vespalib::stringr _file() { _dc.setReadContext(&_readContext); - if (tune.getWantDirectIO()) { + if (tune.getWantDirectIO() && !read_all_upfront) { _file.EnableDirectIO(); } + if (read_all_upfront) { + _file.enableMemoryMap(0); + } if (!_file.OpenReadOnly(name.c_str())) { LOG(error, "could not open %s: %s", _file.GetFileName(), getLastErrorString().c_str()); return; } uint64_t fileSize = _file.getSize(); + uint64_t file_units = DC::file_units(fileSize); _readContext.setFile(&_file); _readContext.setFileSize(fileSize); + bool use_mmap = false; + /* + * Limit memory usage spike by using memory mapped .ssdat file if + * file size is greater than 32 MiB with padding at end of file. + */ + if (read_all_upfront && _file.MemoryMapPtr(0) != nullptr && fileSize >= mmap_file_size_threshold) { + _readContext.reference_compressed_buffer(_file.MemoryMapPtr(0), file_units); + vespalib::FileHeader header; + _dc.readHeader(header, _file.getSize()); + assert(header.hasTag("fileBitSize")); + int64_t file_bit_size = header.getTag("fileBitSize").asInteger(); + use_mmap = DC::is_padded_for_memory_map(file_bit_size, fileSize); + _readContext.setBitOffset(0); + _readContext.setBufferEndFilePos(0); + } if (read_all_upfront) { - _readContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / sizeof(uint64_t), 32_Ki); + if (use_mmap) { + _readContext.reference_compressed_buffer(_file.MemoryMapPtr(0), file_units); + } else { + _readContext.allocComprBuf(file_units, 32_Ki); + } } else { _readContext.allocComprBuf(64_Ki, 32_Ki); } - _dc.emptyBuffer(0); - _readContext.readComprBuffer(); + if (!use_mmap) { + _dc.emptyBuffer(0); + _readContext.readComprBuffer(); + } if (read_all_upfront) { assert(_readContext.getBufferEndFilePos() >= fileSize); } + assert(_dc.getBitPosV() == 0); _valid = true; } @@ -121,7 +147,8 @@ PageDict4FileSeqRead::PageDict4FileSeqRead() _ss(), _sp(), _p(), - _wordNum(0u) + _wordNum(0u), + _mmap_file_size_threshold(32_Mi) { } PageDict4FileSeqRead::~PageDict4FileSeqRead() = default; @@ -166,9 +193,9 @@ bool PageDict4FileSeqRead::open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) { - _ss = std::make_unique<DictFileReadContext>(mySSId, name + ".ssdat", tuneFileRead, true); - _sp = std::make_unique<DictFileReadContext>(mySPId, name + ".spdat", tuneFileRead, false); - _p = std::make_unique<DictFileReadContext>(myPId, name + ".pdat", tuneFileRead, false); + _ss = std::make_unique<DictFileReadContext>(mySSId, name + ".ssdat", tuneFileRead, _mmap_file_size_threshold, true); + _sp = std::make_unique<DictFileReadContext>(mySPId, name + ".spdat", tuneFileRead, _mmap_file_size_threshold, false); + _p = std::make_unique<DictFileReadContext>(myPId, name + ".pdat", tuneFileRead, _mmap_file_size_threshold, false); if ( !_ss->_valid || !_sp->_valid || !_p->_valid ) { return false; } @@ -269,11 +296,9 @@ PageDict4FileSeqWrite::DictFileContext::DictFileContext(bool extended, vespalib: } bool -PageDict4FileSeqWrite::DictFileContext::DictFileContext::close() { - //uint64_t usedPBits = _ec.getWriteOffset(); - _ec.flush(); - _writeContext.writeComprBuffer(true); - +PageDict4FileSeqWrite::DictFileContext::DictFileContext::close() +{ + _ec.pad_for_memory_map_and_flush(); _writeContext.dropComprBuf(); bool success = _file.Sync(); success &= _file.Close(); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index 404f85e9088..40540cd458e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -26,6 +26,7 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead std::unique_ptr<DictFileReadContext> _sp; std::unique_ptr<DictFileReadContext> _p; uint64_t _wordNum; + uint32_t _mmap_file_size_threshold; public: PageDict4FileSeqRead(); ~PageDict4FileSeqRead() override; @@ -38,6 +39,7 @@ public: bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) override; bool close() override; void getParams(index::PostingListParams ¶ms) override; + void set_mmap_file_size_threshold(uint32_t v) { _mmap_file_size_threshold = v; } }; /** diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp index 3654b703648..a513a18ae5d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp @@ -1,8 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "pagedict4randread.h" -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> @@ -33,7 +33,8 @@ PageDict4RandRead::PageDict4RandRead() _pFileBitSize(0u), _ssHeaderLen(0u), _spHeaderLen(0u), - _pHeaderLen(0u) + _pHeaderLen(0u), + _mmap_file_size_threshold(32_Mi) { _ssd.setReadContext(&_ssReadContext); } @@ -229,14 +230,42 @@ PageDict4RandRead::open(const vespalib::string &name, } uint64_t fileSize = _ssfile->getSize(); + uint64_t file_units = DC::file_units(fileSize); _ssReadContext.setFile(_ssfile.get()); _ssReadContext.setFileSize(fileSize); - _ssReadContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / sizeof(uint64_t), 32768u); - _ssd.emptyBuffer(0); - _ssReadContext.readComprBuffer(); - assert(_ssReadContext.getBufferEndFilePos() >= fileSize); + /* + * Limit memory usage spike by using memory mapped .ssdat file if + * file size is greater than 32 MiB with padding at end of file. + * Note: It might cause higher dictionary lookup latencies when + * system is under memory pressure due to pageins. + */ + bool has_read_ss_header = false; + if (_ssfile->MemoryMapPtr(0) != nullptr && fileSize >= _mmap_file_size_threshold) { + _ssReadContext.reference_compressed_buffer(_ssfile->MemoryMapPtr(0), file_units); + assert(_ssd.getReadOffset() == 0u); + readSSHeader(); + has_read_ss_header = true; + } + if (!has_read_ss_header || !DC::is_padded_for_memory_map(_ssFileBitSize, fileSize)) { + /* + * Insufficient padding or small .sdat file. Read whole file into + * memory. + */ + _ssReadContext.allocComprBuf(file_units, 32768u); + _ssd.emptyBuffer(0); + _ssReadContext.setBitOffset(0); + _ssReadContext.setBufferEndFilePos(0); + _ssfile->SetPosition(0); + _ssReadContext.readComprBuffer(); + assert(_ssReadContext.getBufferEndFilePos() >= fileSize); + assert(_ssd.getReadOffset() == 0u); + if (has_read_ss_header) { + _ssReadContext.setPosition(_ssHeaderLen * 8); + } else { + readSSHeader(); + } + } - readSSHeader(); readSPHeader(); readPHeader(); diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h index 051efa486dd..1c2e538cc48 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h @@ -36,6 +36,7 @@ class PageDict4RandRead : public index::DictionaryFileRandRead uint32_t _ssHeaderLen; uint32_t _spHeaderLen; uint32_t _pHeaderLen; + uint32_t _mmap_file_size_threshold; void readSSHeader(); void readSPHeader(); @@ -51,6 +52,7 @@ public: bool close() override; uint64_t getNumWordIds() const override; + void set_mmap_file_size_threshold(uint32_t v) { _mmap_file_size_threshold = v; } }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp index c7480633e21..f2b7911ba55 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp @@ -247,19 +247,7 @@ template <bool bigEndian> void Zc4PostingWriter<bigEndian>::on_close() { - // Write some pad bits to avoid decompression readahead going past - // memory mapped file during search and into SIGSEGV territory. - - // First pad to 64 bits alignment. - _encode_context.smallAlign(64); - _encode_context.writeComprBufferIfNeeded(); - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - _encode_context.padBits(128); - _encode_context.alignDirectIO(); - _encode_context.flush(); - _encode_context.writeComprBuffer(); // Also flushes slack + _encode_context.pad_for_memory_map_and_flush(); } template class Zc4PostingWriter<false>; diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index b0955fe60bd..d19b979c360 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -52,7 +52,7 @@ ConvertRawScoreToCloseness::execute(uint32_t docId) feature_t converted = tfmd->getRawScore(); max_closeness = std::max(max_closeness, converted); } else if (elem.calc) { - feature_t converted = elem.calc->calc_raw_score(docId); + feature_t converted = elem.calc->calc_raw_score<false>(docId); max_closeness = std::max(max_closeness, converted); } } diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 15362b6a224..65a764d8b44 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -12,7 +12,6 @@ #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/stash.h> -#include <cmath> #include <limits> #include <vespa/log/log.h> @@ -62,7 +61,7 @@ ConvertRawscoreToDistance::execute(uint32_t docId) feature_t converted = elem.calc ? elem.calc->function().to_distance(invdist) : ((1.0 / invdist) - 1.0); min_distance = std::min(min_distance, converted); } else if (elem.calc) { - feature_t invdist = elem.calc->calc_raw_score(docId); + feature_t invdist = elem.calc->calc_raw_score<false>(docId); feature_t converted = elem.calc->function().to_distance(invdist); min_distance = std::min(min_distance, converted); } @@ -130,7 +129,10 @@ GeoGCDExecutor::GeoGCDExecutor(GeoLocationSpecPtrs locations, const attribute::I : FeatureExecutor(), _locations(), _pos(pos), - _intBuf() + _intBuf(), + _best_index(0.0), + _best_lat(0.0), + _best_lng(0.0) { if (_pos == nullptr) { return; @@ -140,7 +142,7 @@ GeoGCDExecutor::GeoGCDExecutor(GeoLocationSpecPtrs locations, const attribute::I if (p && p->location.valid() && p->location.has_point) { double lat = p->location.point.y / 1.0e6; double lng = p->location.point.x / 1.0e6; - _locations.emplace_back(search::common::GeoGcd{lat, lng}); + _locations.emplace_back(lat, lng); } } } diff --git a/searchlib/src/vespa/searchlib/fef/query_value.cpp b/searchlib/src/vespa/searchlib/fef/query_value.cpp index e7791843992..78f2e06edf7 100644 --- a/searchlib/src/vespa/searchlib/fef/query_value.cpp +++ b/searchlib/src/vespa/searchlib/fef/query_value.cpp @@ -12,6 +12,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/locale/c.h> #include <vespa/vespalib/util/issue.h> +#include <vespa/vespalib/objects/nbostream.h> #include <cerrno> using document::TensorDataType; diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 42fa7e54581..a76185cbd66 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/issue.h> #include <vespa/vespalib/util/execution_profiler.h> +#include <vespa/vespalib/objects/nbostream.h> #include <algorithm> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/fef/utils.cpp b/searchlib/src/vespa/searchlib/fef/utils.cpp index c3ef4426f53..1b660ba380e 100644 --- a/searchlib/src/vespa/searchlib/fef/utils.cpp +++ b/searchlib/src/vespa/searchlib/fef/utils.cpp @@ -3,6 +3,7 @@ #include "utils.h" #include "rank_program.h" #include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vector> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp index 75dc26fe54c..dc47d306767 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistparams.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistparams.cpp @@ -91,21 +91,27 @@ template void PostingListParams::get<bool>(const vespalib::string &key, bool &val) const; template void -PostingListParams::set<int32_t>(const vespalib::string &key, const int32_t &val); +PostingListParams::set<int>(const vespalib::string& key, const int& val); template void -PostingListParams::get<int32_t>(const vespalib::string &key, int32_t &val) const; +PostingListParams::get<int>(const vespalib::string& key, int& val) const; template void -PostingListParams::set<uint32_t>(const vespalib::string &key, const uint32_t &val); +PostingListParams::set<unsigned int>(const vespalib::string& key, const unsigned int& val); template void -PostingListParams::get<uint32_t>(const vespalib::string &key, uint32_t &val) const; +PostingListParams::get<unsigned int>(const vespalib::string& key, unsigned int& val) const; template void -PostingListParams::set<uint64_t>(const vespalib::string &key, const uint64_t &val); +PostingListParams::set<unsigned long>(const vespalib::string& key, const unsigned long& val); template void -PostingListParams::get<uint64_t>(const vespalib::string &key, uint64_t &val) const; +PostingListParams::get<unsigned long>(const vespalib::string& key, unsigned long& val) const; + +template void +PostingListParams::set<unsigned long long>(const vespalib::string& key, const unsigned long long& val); + +template void +PostingListParams::get<unsigned long long>(const vespalib::string& key, unsigned long long& val) const; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index eba135d8519..2bc94073c92 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -261,7 +261,7 @@ public: queryeval::FlowStats calculate_flow_stats(uint32_t docid_limit) const override { double rel_est = abs_to_rel_est(_posting_itr.size(), docid_limit); - return {rel_est, btree_cost(), btree_strict_cost(rel_est)}; + return {rel_est, btree_cost(rel_est), btree_strict_cost(rel_est)}; } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray& tfmda) const override { diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index 5e3b1dffe3a..a05c68bea46 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -87,6 +87,8 @@ public: IFLAG_NORANK = 0x00000001, // this term should not be ranked (not exposed to rank framework) IFLAG_SPECIALTOKEN = 0x00000002, IFLAG_NOPOSITIONDATA = 0x00000004, // we should not use position data when ranking this term + IFLAG_FILTER = 0x00000008, // see GetCreator(flags) below + IFLAG_PREFIX_MATCH = 0x00000010 }; /** Extra information on each item (creator id) coded in bit 3 of flags */ diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index d15e1ca0512..5486a677026 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -113,9 +113,18 @@ public: uint32_t getUniqueId() const noexcept { return _currUniqueId; } // Get the flags of the current item. - bool hasNoRankFlag() const noexcept { return (_currFlags & ParseItem::IFLAG_NORANK) != 0; } - bool hasSpecialTokenFlag() const noexcept { return (_currFlags & ParseItem::IFLAG_SPECIALTOKEN) != 0; } - bool hasNoPositionDataFlag() const noexcept { return (_currFlags & ParseItem::IFLAG_NOPOSITIONDATA) != 0; } + [[nodiscard]] bool hasNoRankFlag() const noexcept { + return (_currFlags & ParseItem::IFLAG_NORANK) != 0; + } + [[nodiscard]] bool hasSpecialTokenFlag() const noexcept { + return (_currFlags & ParseItem::IFLAG_SPECIALTOKEN) != 0; + } + [[nodiscard]] bool hasNoPositionDataFlag() const noexcept { + return (_currFlags & ParseItem::IFLAG_NOPOSITIONDATA) != 0; + } + [[nodiscard]] bool has_prefix_match_semantics() const noexcept { + return (_currFlags & ParseItem::IFLAG_PREFIX_MATCH) != 0; + } uint32_t getArity() const noexcept { return _currArity; } @@ -127,16 +136,16 @@ public: bool getAllowApproximate() const noexcept { return (_extraIntArg2 != 0); } uint32_t getExploreAdditionalHits() const noexcept { return _extraIntArg3; } - // fuzzy match arguments - uint32_t getFuzzyMaxEditDistance() const noexcept { return _extraIntArg1; } - uint32_t getFuzzyPrefixLength() const noexcept { return _extraIntArg2; } + // fuzzy match arguments (see also: has_prefix_match_semantics() for fuzzy prefix matching) + [[nodiscard]] uint32_t fuzzy_max_edit_distance() const noexcept { return _extraIntArg1; } + [[nodiscard]] uint32_t fuzzy_prefix_lock_length() const noexcept { return _extraIntArg2; } std::unique_ptr<query::PredicateQueryTerm> getPredicateQueryTerm(); std::unique_ptr<query::TermVector> get_terms(); vespalib::stringref getIndexName() const noexcept { return _curr_index_name; } vespalib::stringref getTerm() const noexcept { return _curr_term; } - int64_t getIntergerTerm() const noexcept { return _curr_integer_term; } + int64_t getIntegerTerm() const noexcept { return _curr_integer_term; } static vespalib::stringref DEFAULT_INDEX; }; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index 7ff796e5b7d..8159d5c4147 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -50,6 +50,7 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig return; } size_t buffer_size = 21 * 2 + 3 + key.size(); // 2 numbers + punctuation + key + // GNU extension: Variable-length automatic array char buffer[buffer_size]; int size; int prefix_size = snprintf(buffer, buffer_size, "%s=", key.c_str()); diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.h b/searchlib/src/vespa/searchlib/predicate/simple_index.h index 0e3c9828b21..b117bf188ed 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.h +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.h @@ -166,6 +166,7 @@ private: double ratio, size_t vector_length) const; double getDocumentRatio(size_t document_count, uint32_t doc_id_limit) const; size_t getDocumentCount(vespalib::datastore::EntryRef ref) const; + size_t get_frozen_document_count(vespalib::datastore::EntryRef ref) const; bool shouldCreateVectorPosting(size_t size, double ratio) const; bool shouldRemoveVectorPosting(size_t size, double ratio) const; size_t getVectorPostingSize(const PostingVector &vector) const { @@ -226,7 +227,7 @@ template<typename FunctionType> void SimpleIndex<Posting, Key, DocId>::foreach_frozen_key(vespalib::datastore::EntryRef ref, Key key, FunctionType func) const { auto it = _vector_posting_lists.getFrozenView().find(key); - double ratio = getDocumentRatio(getDocumentCount(ref), _limit_provider.getDocIdLimit()); + double ratio = getDocumentRatio(get_frozen_document_count(ref), _limit_provider.getDocIdLimit()); if (it.valid() && ratio > _config.foreach_vector_threshold) { auto &vector = *it.getData(); size_t size = getVectorPostingSize(vector); diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp index 0516227081e..b1a6f9b1a49 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp @@ -253,6 +253,12 @@ SimpleIndex<Posting, Key, DocId>::getDocumentCount(vespalib::datastore::EntryRef }; template <typename Posting, typename Key, typename DocId> +size_t +SimpleIndex<Posting, Key, DocId>::get_frozen_document_count(vespalib::datastore::EntryRef ref) const { + return _btree_posting_lists.frozenSize(ref); +}; + +template <typename Posting, typename Key, typename DocId> bool SimpleIndex<Posting, Key, DocId>::shouldRemoveVectorPosting(size_t size, double ratio) const { return size < _config.lower_vector_size_threshold || ratio < _config.lower_docid_freq_threshold; diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp index 060cd5015b3..09fc443cf0e 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp @@ -248,10 +248,11 @@ QueryTermSimple::QueryTermSimple(const string & term_, Type type) _type(type), _diversityCutoffStrict(false), _valid(true), + _fuzzy_prefix_match(false), _term(term_), _diversityAttribute(), - _fuzzyMaxEditDistance(2), - _fuzzyPrefixLength(0) + _fuzzy_max_edit_distance(2), + _fuzzy_prefix_lock_length(0) { if (isFullRange(_term)) { stringref rest(_term.c_str() + 1, _term.size() - 2); diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.h b/searchlib/src/vespa/searchlib/query/query_term_simple.h index a740afb0340..b0ee7aa54bc 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.h +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.h @@ -44,8 +44,9 @@ public: size_t getDiversityCutoffGroups() const noexcept { return _diversityCutoffGroups; } bool getDiversityCutoffStrict() const noexcept { return _diversityCutoffStrict; } vespalib::stringref getDiversityAttribute() const noexcept { return _diversityAttribute; } - size_t getFuzzyMaxEditDistance() const noexcept { return _fuzzyMaxEditDistance; } - size_t getFuzzyPrefixLength() const noexcept { return _fuzzyPrefixLength; } + [[nodiscard]] size_t fuzzy_max_edit_distance() const noexcept { return _fuzzy_max_edit_distance; } + [[nodiscard]] size_t fuzzy_prefix_lock_length() const noexcept { return _fuzzy_prefix_lock_length; } + [[nodiscard]] bool fuzzy_prefix_match() const noexcept { return _fuzzy_prefix_match; } bool getAsIntegerTerm(int64_t & lower, int64_t & upper) const noexcept; bool getAsFloatTerm(double & lower, double & upper) const noexcept; bool getAsFloatTerm(float & lower, float & upper) const noexcept; @@ -77,14 +78,17 @@ private: Type _type; bool _diversityCutoffStrict; bool _valid; +protected: + bool _fuzzy_prefix_match; // set in QueryTerm +private: string _term; stringref _diversityAttribute; template <typename T, typename D> bool getAsNumericTerm(T & lower, T & upper, D d) const noexcept; protected: - uint32_t _fuzzyMaxEditDistance; // set in QueryTerm - uint32_t _fuzzyPrefixLength; + uint32_t _fuzzy_max_edit_distance; // set in QueryTerm + uint32_t _fuzzy_prefix_lock_length; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.cpp b/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.cpp index f33fe44369a..8a4ec184de8 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.cpp @@ -13,20 +13,21 @@ constexpr bool normalizing_implies_cased(Normalizing norm) noexcept { FuzzyTerm::FuzzyTerm(std::unique_ptr<QueryNodeResultBase> result_base, stringref term, const string& index, Type type, Normalizing normalizing, - uint8_t max_edits, uint32_t prefix_size) + uint8_t max_edits, uint32_t prefix_lock_length, bool prefix_match) : QueryTerm(std::move(result_base), term, index, type, normalizing), _dfa_matcher(), _fallback_matcher() { - setFuzzyMaxEditDistance(max_edits); - setFuzzyPrefixLength(prefix_size); + set_fuzzy_max_edit_distance(max_edits); + set_fuzzy_prefix_lock_length(prefix_lock_length); + set_fuzzy_prefix_match(prefix_match); std::string_view term_view(term.data(), term.size()); const bool cased = normalizing_implies_cased(normalizing); if (attribute::DfaFuzzyMatcher::supports_max_edits(max_edits)) { - _dfa_matcher = std::make_unique<attribute::DfaFuzzyMatcher>(term_view, max_edits, prefix_size, cased); + _dfa_matcher = std::make_unique<attribute::DfaFuzzyMatcher>(term_view, max_edits, prefix_lock_length, cased, prefix_match); } else { - _fallback_matcher = std::make_unique<vespalib::FuzzyMatcher>(term_view, max_edits, prefix_size, cased); + _fallback_matcher = std::make_unique<vespalib::FuzzyMatcher>(term_view, max_edits, prefix_lock_length, cased, prefix_match); } } diff --git a/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.h b/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.h index c6c88b18969..552a839cb3b 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.h +++ b/searchlib/src/vespa/searchlib/query/streaming/fuzzy_term.h @@ -23,7 +23,7 @@ class FuzzyTerm : public QueryTerm { public: FuzzyTerm(std::unique_ptr<QueryNodeResultBase> result_base, stringref term, const string& index, Type type, Normalizing normalizing, - uint8_t max_edits, uint32_t prefix_size); + uint8_t max_edits, uint32_t prefix_lock_length, bool prefix_match); ~FuzzyTerm() override; [[nodiscard]] FuzzyTerm* as_fuzzy_term() noexcept override { return this; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 94a479fd2d3..6e0a5df1dcf 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -113,7 +113,7 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor QueryTerm::string ssTerm; if (type == ParseItem::ITEM_PURE_WEIGHTED_LONG) { char buf[24]; - auto res = std::to_chars(buf, buf + sizeof(buf), queryRep.getIntergerTerm(), 10); + auto res = std::to_chars(buf, buf + sizeof(buf), queryRep.getIntegerTerm(), 10); ssTerm.assign(buf, res.ptr - buf); } else { ssTerm = queryRep.getTerm(); @@ -130,7 +130,8 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor qt = std::make_unique<RegexpTerm>(factory.create(), ssTerm, ssIndex, TermType::REGEXP, normalize_mode); } else if (sTerm == TermType::FUZZYTERM) { qt = std::make_unique<FuzzyTerm>(factory.create(), ssTerm, ssIndex, TermType::FUZZYTERM, normalize_mode, - queryRep.getFuzzyMaxEditDistance(), queryRep.getFuzzyPrefixLength()); + queryRep.fuzzy_max_edit_distance(), queryRep.fuzzy_prefix_lock_length(), + queryRep.has_prefix_match_semantics()); } else [[likely]] { qt = std::make_unique<QueryTerm>(factory.create(), ssTerm, ssIndex, sTerm, normalize_mode); } @@ -236,7 +237,7 @@ QueryNode::populate_multi_term(Normalizing string_normalize_mode, MultiTerm& mt, break; case ParseItem::ITEM_PURE_WEIGHTED_LONG: { - auto res = std::to_chars(buf, buf + sizeof(buf), queryRep.getIntergerTerm(), 10); + auto res = std::to_chars(buf, buf + sizeof(buf), queryRep.getIntegerTerm(), 10); subterm.assign(buf, res.ptr - buf); term = std::make_unique<QueryTerm>(std::unique_ptr<QueryNodeResultBase>(), subterm, "", QueryTermSimple::Type::WORD, Normalizing::NONE); diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h index 05b12804d52..0e0ab98295b 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h @@ -100,8 +100,9 @@ public: void visitMembers(vespalib::ObjectVisitor &visitor) const override; void setIndex(const string & index_) override { _index = index_; } const string & getIndex() const override { return _index; } - void setFuzzyMaxEditDistance(uint32_t fuzzyMaxEditDistance) { _fuzzyMaxEditDistance = fuzzyMaxEditDistance; } - void setFuzzyPrefixLength(uint32_t fuzzyPrefixLength) { _fuzzyPrefixLength = fuzzyPrefixLength; } + void set_fuzzy_max_edit_distance(uint32_t fuzzy_max_edit_distance) noexcept { _fuzzy_max_edit_distance = fuzzy_max_edit_distance; } + void set_fuzzy_prefix_lock_length(uint32_t fuzzy_prefix_length) noexcept { _fuzzy_prefix_lock_length = fuzzy_prefix_length; } + void set_fuzzy_prefix_match(bool prefix_match) noexcept { _fuzzy_prefix_match = prefix_match; } virtual NearestNeighborQueryNode* as_nearest_neighbor_query_node() noexcept; virtual MultiTerm* as_multi_term() noexcept; virtual const MultiTerm* as_multi_term() const noexcept; diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 41990af6908..a498076bb09 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -226,8 +226,8 @@ create_nearest_neighbor_term(vespalib::stringref query_tensor_name, vespalib::st template <class NodeTypes> typename NodeTypes::FuzzyTerm * createFuzzyTerm(vespalib::stringref term, vespalib::stringref view, int32_t id, Weight weight, - uint32_t maxEditDistance, uint32_t prefixLength) { - return new typename NodeTypes::FuzzyTerm(term, view, id, weight, maxEditDistance, prefixLength); + uint32_t max_edit_distance, uint32_t prefix_lock_length, bool prefix_match) { + return new typename NodeTypes::FuzzyTerm(term, view, id, weight, max_edit_distance, prefix_lock_length, prefix_match); } template <class NodeTypes> @@ -343,9 +343,10 @@ public: return addTerm(createRegExpTerm<NodeTypes>(term, view, id, weight)); } typename NodeTypes::FuzzyTerm &addFuzzyTerm(stringref term, stringref view, int32_t id, Weight weight, - uint32_t maxEditDistance, uint32_t prefixLength) { + uint32_t max_edit_distance, uint32_t prefix_lock_length, + bool prefix_match) { adjustWeight(weight); - return addTerm(createFuzzyTerm<NodeTypes>(term, view, id, weight, maxEditDistance, prefixLength)); + return addTerm(createFuzzyTerm<NodeTypes>(term, view, id, weight, max_edit_distance, prefix_lock_length, prefix_match)); } typename NodeTypes::NearestNeighborTerm &add_nearest_neighbor_term(stringref query_tensor_name, stringref field_name, int32_t id, Weight weight, uint32_t target_num_hits, diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index f578bb57e21..116e677d439 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -201,7 +201,8 @@ private: replicate(node, _builder.addFuzzyTerm( node.getTerm(), node.getView(), node.getId(), node.getWeight(), - node.getMaxEditDistance(), node.getPrefixLength())); + node.max_edit_distance(), node.prefix_lock_length(), + node.prefix_match())); } std::unique_ptr<TermVector> replicate_subterms(const InTerm& original) { diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index 535402bf14d..34bc39ae504 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -163,8 +163,10 @@ struct SimpleNearestNeighborTerm : NearestNeighborTerm { struct SimpleFuzzyTerm : FuzzyTerm { SimpleFuzzyTerm(const Type &term, vespalib::stringref view, int32_t id, Weight weight, - uint32_t maxEditDistance, uint32_t prefixLength) - : FuzzyTerm(term, view, id, weight, maxEditDistance, prefixLength) { + uint32_t max_edit_distance, uint32_t prefix_lock_length, + bool prefix_match) + : FuzzyTerm(term, view, id, weight, max_edit_distance, prefix_lock_length, prefix_match) + { } ~SimpleFuzzyTerm() override; }; diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 6707712ee69..b71a86a214a 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -225,6 +225,9 @@ class QueryNodeConverter : public QueryVisitor { if (!node.usePositionData()) { flags |= ParseItem::IFLAG_NOPOSITIONDATA; } + if (node.prefix_match()) { + flags |= ParseItem::IFLAG_PREFIX_MATCH; + } if (flags != 0) { features |= ParseItem::IF_FLAGS; } @@ -289,8 +292,8 @@ class QueryNodeConverter : public QueryVisitor { void visit(FuzzyTerm &node) override { createTerm(node, ParseItem::ITEM_FUZZY); - appendCompressedPositiveNumber(node.getMaxEditDistance()); - appendCompressedPositiveNumber(node.getPrefixLength()); + appendCompressedPositiveNumber(node.max_edit_distance()); + appendCompressedPositiveNumber(node.prefix_lock_length()); } void visit(NearestNeighborTerm &node) override { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.cpp index 4ff05360ded..e3642294a55 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.cpp @@ -17,7 +17,7 @@ StackDumpQueryCreatorHelper::populateMultiTerm(SimpleQueryStackDumpIterator &que ParseItem::ItemType type = queryStack.getType(); switch (type) { case ParseItem::ITEM_PURE_WEIGHTED_LONG: - mt.addTerm(queryStack.getIntergerTerm(), queryStack.GetWeight()); + mt.addTerm(queryStack.getIntegerTerm(), queryStack.GetWeight()); break; case ParseItem::ITEM_PURE_WEIGHTED_STRING: mt.addTerm(queryStack.getTerm(), queryStack.GetWeight()); diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 7c743a72567..52466b5cd13 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -42,6 +42,9 @@ public: if (queryStack.hasNoPositionDataFlag()) { t->setPositionData(false); } + if (queryStack.has_prefix_match_semantics()) [[unlikely]] { + t->set_prefix_match(true); + } } } if (builder.hasError()) { @@ -156,7 +159,7 @@ private: t = &builder.addStringTerm(term, pureTermView, id, weight); } else if (type == ParseItem::ITEM_PURE_WEIGHTED_LONG) { char buf[24]; - auto res = std::to_chars(buf, buf + sizeof(buf), queryStack.getIntergerTerm(), 10); + auto res = std::to_chars(buf, buf + sizeof(buf), queryStack.getIntegerTerm(), 10); t = &builder.addNumberTerm(vespalib::stringref(buf, res.ptr - buf), pureTermView, id, weight); } else if (type == ParseItem::ITEM_PREFIXTERM) { t = &builder.addPrefixTerm(term, view, id, weight); @@ -185,9 +188,10 @@ private: } else if (type == ParseItem::ITEM_REGEXP) { t = &builder.addRegExpTerm(term, view, id, weight); } else if (type == ParseItem::ITEM_FUZZY) { - uint32_t maxEditDistance = queryStack.getFuzzyMaxEditDistance(); - uint32_t prefixLength = queryStack.getFuzzyPrefixLength(); - t = &builder.addFuzzyTerm(term, view, id, weight, maxEditDistance, prefixLength); + uint32_t max_edit_distance = queryStack.fuzzy_max_edit_distance(); + uint32_t prefix_lock_length = queryStack.fuzzy_prefix_lock_length(); + bool prefix_match = queryStack.has_prefix_match_semantics(); + t = &builder.addFuzzyTerm(term, view, id, weight, max_edit_distance, prefix_lock_length, prefix_match); } else if (type == ParseItem::ITEM_STRING_IN) { t = &builder.add_in_term(queryStack.get_terms(), MultiTerm::Type::STRING, view, id, weight); } else if (type == ParseItem::ITEM_NUMERIC_IN) { diff --git a/searchlib/src/vespa/searchlib/query/tree/term.cpp b/searchlib/src/vespa/searchlib/query/tree/term.cpp index 45ba259a799..35f6ea5c8f5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/term.cpp @@ -12,12 +12,14 @@ Term::Term(vespalib::stringref view, int32_t id, Weight weight) : _id(id), _weight(weight), _ranked(true), - _position_data(true) + _position_data(true), + _prefix_match(false) { } void Term::setStateFrom(const Term& other) { setRanked(other.isRanked()); setPositionData(other.usePositionData()); + set_prefix_match(other.prefix_match()); // too late to copy this state: assert(_view == other.getView()); assert(_id == other.getId()); diff --git a/searchlib/src/vespa/searchlib/query/tree/term.h b/searchlib/src/vespa/searchlib/query/tree/term.h index 2f57c3cb06d..678d018d111 100644 --- a/searchlib/src/vespa/searchlib/query/tree/term.h +++ b/searchlib/src/vespa/searchlib/query/tree/term.h @@ -18,6 +18,7 @@ class Term Weight _weight; bool _ranked; bool _position_data; + bool _prefix_match; public: virtual ~Term() = 0; @@ -25,14 +26,17 @@ public: void setView(const vespalib::string & view) { _view = view; } void setRanked(bool ranked) noexcept { _ranked = ranked; } void setPositionData(bool position_data) noexcept { _position_data = position_data; } + // Used for fuzzy prefix matching. Not to be confused with distinct Prefix query term type + void set_prefix_match(bool prefix_match) noexcept { _prefix_match = prefix_match; } void setStateFrom(const Term& other); const vespalib::string & getView() const noexcept { return _view; } Weight getWeight() const noexcept { return _weight; } int32_t getId() const noexcept { return _id; } - bool isRanked() const noexcept { return _ranked; } - bool usePositionData() const noexcept { return _position_data; } + [[nodiscard]] bool isRanked() const noexcept { return _ranked; } + [[nodiscard]] bool usePositionData() const noexcept { return _position_data; } + [[nodiscard]] bool prefix_match() const noexcept { return _prefix_match; } static bool isPossibleRangeTerm(vespalib::stringref term) noexcept { return (term[0] == '[' || term[0] == '<' || term[0] == '>'); diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index 97e659dcc77..42dc0717368 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -119,19 +119,22 @@ public: //----------------------------------------------------------------------------- class FuzzyTerm : public QueryNodeMixin<FuzzyTerm, StringBase> { -private: - uint32_t _maxEditDistance; - uint32_t _prefixLength; + uint32_t _max_edit_distance; + uint32_t _prefix_lock_length; + // Prefix match mode is stored in parent Term public: FuzzyTerm(const Type &term, vespalib::stringref view, - int32_t id, Weight weight, uint32_t maxEditDistance, uint32_t prefixLength) - : QueryNodeMixinType(term, view, id, weight), - _maxEditDistance(maxEditDistance), - _prefixLength(prefixLength) - {} + int32_t id, Weight weight, uint32_t max_edit_distance, + uint32_t prefix_lock_length, bool prefix_match) + : QueryNodeMixinType(term, view, id, weight), + _max_edit_distance(max_edit_distance), + _prefix_lock_length(prefix_lock_length) + { + set_prefix_match(prefix_match); + } - uint32_t getMaxEditDistance() const { return _maxEditDistance; } - uint32_t getPrefixLength() const { return _prefixLength; } + [[nodiscard]] uint32_t max_edit_distance() const { return _max_edit_distance; } + [[nodiscard]] uint32_t prefix_lock_length() const { return _prefix_lock_length; } virtual ~FuzzyTerm() = 0; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 43339b68999..d11ee25a7e5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -619,7 +619,7 @@ IntermediateBlueprint::sort(InFlow in_flow) { resolve_strict(in_flow); if (!opt_keep_order()) [[likely]] { - sort(_children, in_flow.strict(), opt_sort_by_cost()); + sort(_children, in_flow); } auto flow = my_flow(in_flow); for (size_t i = 0; i < _children.size(); ++i) { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 1501289c590..a443f34f856 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -499,7 +499,7 @@ public: virtual HitEstimate combine(const std::vector<HitEstimate> &data) const = 0; virtual FieldSpecBaseList exposeFields() const = 0; - virtual void sort(Children &children, bool strict, bool sort_by_cost) const = 0; + virtual void sort(Children &children, InFlow in_flow) const = 0; virtual SearchIteratorUP createIntermediateSearch(MultiSearch::Children subSearches, fef::MatchData &md) const = 0; diff --git a/searchlib/src/vespa/searchlib/queryeval/flow.h b/searchlib/src/vespa/searchlib/queryeval/flow.h index b57e823f4d7..0dc92a6cf88 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow.h @@ -195,63 +195,40 @@ double ordered_cost_of(ADAPTER adapter, const T &children, F flow, bool allow_fo return total_cost; } -size_t select_strict_and_child(auto adapter, const auto &children) { - double est = 1.0; - double cost = 0.0; - size_t best_idx = 0; - double best_diff = std::numeric_limits<double>::max(); - for (size_t idx = 0; idx < children.size(); ++idx) { - auto child = FlowStats::from(adapter, children[idx]); - double child_abs_cost = est * child.cost; - double my_diff = (child.strict_cost + child.estimate * cost) - (cost + child_abs_cost); - if (my_diff < best_diff) { - best_diff = my_diff; - best_idx = idx; - } - cost += child_abs_cost; - est *= child.estimate; - } - return best_idx; -} - -auto select_forced_strict_and_child_impl(auto adapter, const auto &children, size_t first, auto ordered, auto destructive) { - auto est_until = [&](size_t limit)noexcept{ - double my_est = 1.0; - for (size_t i = 0; i < limit; ++i) { - my_est *= adapter.estimate(children[i]); - } - return my_est; - }; - double est = est_until(first); +auto select_strict_and_child(auto adapter, const auto &children, size_t first, double est, bool native_strict) { double cost = 0.0; size_t best_idx = first; size_t best_target = first; double best_diff = std::numeric_limits<double>::max(); + for (size_t i = 0; i < first; ++i) { + est *= adapter.estimate(children[i]); + } + double first_est = est; for (size_t idx = first; idx < children.size(); ++idx) { auto child = FlowStats::from(adapter, children[idx]); double child_abs_cost = est * child.cost; - double forced_cost = forced_strict_cost(child, est); - double my_diff = (forced_cost + child.estimate * cost) - (cost + child_abs_cost); + double child_strict_cost = (first == 0 && native_strict) ? child.strict_cost : forced_strict_cost(child, first_est); + double my_diff = (child_strict_cost + child.estimate * cost) - (cost + child_abs_cost); size_t target = first; - while (ordered && target > 0) { + while (target > 0) { size_t candidate = target - 1; auto other = FlowStats::from(adapter, children[candidate]); if (other.estimate < child.estimate) { // do not move past someone with lower estimate break; } - if (!destructive && !should_force_strict(other, (est_until(candidate) * child.estimate))) { - // no not move past someone who will lose strictness, - // unless it is explicitly allowed by enabling - // destructive (re-)ordering - break; - } target = candidate; my_diff += (0.2 * child.estimate - 0.2 * other.estimate); - if (candidate == 0) { + if (candidate == 0 && native_strict) { // the first iterator produces its own in-flow my_diff += (0.2 * child.estimate - 0.2 * other.estimate); } + // note that 'my_diff' might overestimate the cost + // (underestimate the benefit) of inserting 'child' before + // 'other' if it leads to 'other' becoming + // non-strict. This will also leave 'other' in a + // potentially unoptimal location. Unit tests indicate + // that the effects of this are minor. } if (my_diff < best_diff) { best_diff = my_diff; @@ -264,19 +241,6 @@ auto select_forced_strict_and_child_impl(auto adapter, const auto &children, siz return std::make_tuple(best_idx, best_target, best_diff); } -auto select_forced_strict_and_child(auto adapter, const auto &children, size_t first) { - auto [idx, target, diff] = select_forced_strict_and_child_impl(adapter, children, first, std::false_type{}, std::false_type{}); - return std::make_pair(idx, diff); -} - -auto select_forced_strict_and_child_with_order(auto adapter, const auto &children, size_t first) { - return select_forced_strict_and_child_impl(adapter, children, first, std::true_type{}, std::false_type{}); -} - -auto select_forced_strict_and_child_with_destructive_order(auto adapter, const auto &children, size_t first) { - return select_forced_strict_and_child_impl(adapter, children, first, std::true_type{}, std::true_type{}); -} - } // flow template <typename FLOW> @@ -313,11 +277,28 @@ public: static double estimate_of(const auto &children) { return estimate_of(flow::make_adapter(children), children); } + // assume children are already ordered by calling sort (with same strictness as in_flow) + static void reorder_for_extra_strictness(auto adapter, auto &children, InFlow in_flow, size_t max_extra) { + size_t num_strict = in_flow.strict() ? 1 : 0; + size_t max_strict = num_strict + max_extra; + for (size_t next = num_strict; (next < max_strict) && (next < children.size()); ++next) { + auto [idx, target, diff] = flow::select_strict_and_child(adapter, children, next, in_flow.rate(), in_flow.strict()); + if (diff >= 0.0) { + break; + } + auto pos = children.begin() + idx; + std::rotate(children.begin() + target, pos, pos + 1); + } + } + static void reorder_for_extra_strictness(auto &children, InFlow in_flow, size_t max_extra) { + reorder_for_extra_strictness(flow::make_adapter(children), children, in_flow, max_extra); + } static void sort(auto adapter, auto &children, bool strict) { flow::sort<flow::MinAndCost>(adapter, children); if (strict && children.size() > 1) { - auto pos = children.begin() + flow::select_strict_and_child(adapter, children); - std::rotate(children.begin(), pos, pos + 1); + auto [idx, target, ignore_diff] = flow::select_strict_and_child(adapter, children, 0, 1.0, true); + auto pos = children.begin() + idx; + std::rotate(children.begin() + target, pos, pos + 1); } } static void sort(auto &children, bool strict) { diff --git a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h index 51e544b2e30..6389de35264 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow_tuning.h @@ -1,58 +1,132 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/searchcommon/attribute/basictype.h> +#include <vespa/searchcommon/attribute/collectiontype.h> #include <cmath> #include <cstddef> namespace search::queryeval::flow { -inline double heap_cost(double my_est, size_t num_children) { - return my_est * std::log2(std::max(size_t(1),num_children)); -} - /** - * The following constants and formulas were derived after analyzing term search over attributes - * (with and without fast-search) and disk index by using the iterator benchmark program: + * The following constants and formulas were derived after benchmarking and analyzing + * using the following benchmark program: * searchlib/src/tests/queryeval/iterator_benchmark * - * The following tests were executed on a machine with an Intel Xeon 2.5 GHz CPU with 48 cores and 256 Gb of memory: - * ./searchlib_iterator_benchmark_test_app --gtest_filter='*analyze_term_search*' - * - * TODO: Add details on OR benchmarking. + * The tests were executed on: + * - Machine with an Intel Xeon 2.5 GHz CPU with 48 cores and 256 Gb of memory. + * - Apple M1 MacBook Pro (2021) 32 GB. * * The benchmark summary shows the 'average ms per cost' of the different benchmark cases. - * The following constants and formulas were derived to balance 'average ms per cost' to be similar + * The constants and formulas are adjusted to balance 'average ms per cost' to be similar * across the different benchmark cases. + * + * The AND benchmark cases outputs the ratio (esti/forc) of the time_ms used by two query planning algorithms: + * 'estimate' (legacy) and 'cost with allowed force strict' (new). + * 'max_speedup' indicates the gain of using the new cost model, while 'min_speedup' indicates the loss. + * The constants and formulas are also adjusted to maximize speedup, while reducing loss. + * Tests used: + * - IteratorBenchmark::analyze_AND_filter_vs_IN + * - IteratorBenchmark::analyze_AND_filter_vs_OR + * - IteratorBenchmark::analyze_AND_filter_vs_IN_array + * - IteratorBenchmark::analyze_AND_bitvector_vs_IN + */ + +/** + * This function is used when calculating the strict cost of + * intermediate and complex leaf blueprints that use a heap for their strict iterator implementation. + * Tests used: + * - IteratorBenchmark::analyze_IN_strict + * - IteratorBenchmark::analyze_OR_strict + */ +inline double heap_cost(double my_est, size_t num_children) { + return my_est * std::log2(std::max(size_t(1), num_children)); +} + +/** + * Returns the number of memory indirections needed when doing lookups + * in an attribute with the given type. */ +inline size_t get_num_indirections(const attribute::BasicType& basic_type, + const attribute::CollectionType& col_type) { + size_t res = 0; + if (basic_type == attribute::BasicType::STRING) { + res += 1; + } + if (col_type != attribute::CollectionType::SINGLE) { + res += 1; + } + return res; +} // Non-strict cost of lookup based matching in an attribute (not fast-search). +// Test used: IteratorBenchmark::analyze_term_search_in_attributes_non_strict inline double lookup_cost(size_t num_indirections) { return 1.0 + (num_indirections * 1.0); } +// Non-strict cost of reverse lookup into a hash table (containing terms from a multi-term operator). +// Test used: IteratorBenchmark::analyze_IN_non_strict +inline double reverse_hash_lookup() { + return 1.0; +} + // Strict cost of lookup based matching in an attribute (not fast-search). +// IteratorBenchmark::analyze_term_search_in_attributes_strict inline double lookup_strict_cost(size_t num_indirections) { return lookup_cost(num_indirections); } -// Non-strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). -inline double btree_cost() { - return 1.0; +/** + * Estimates the cost of evaluating an always strict iterator (e.g. btree) in a non-strict context. + * + * When the estimate and strict cost is low, this models the cost of checking whether + * the seek docid matches the docid the iterator is already positioned at (the 0.2 factor). + * + * The resulting non-strict cost is most accurate when the inflow is 1.0. + * The resulting non-strict cost is highly underestimated when the inflow goes to 0.0. + * It is important to have a better estimate at higher inflows, + * as the latency (time) penalty is higher if choosing wrong. + * + * Note: This formula is equal to forced_strict_cost() in flow.h. + */ +inline double non_strict_cost_of_strict_iterator(double estimate, double strict_cost) { + return 0.2 * (1.0 - estimate) + strict_cost; } // Strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). +// Test used: IteratorBenchmark::analyze_term_search_in_fast_search_attributes inline double btree_strict_cost(double my_est) { return my_est; } -// Non-strict cost of matching in a disk index posting list. -inline double disk_index_cost() { - return 1.5; +// Non-strict cost of matching in a btree posting list (e.g. fast-search attribute or memory index field). +// Test used: IteratorBenchmark::analyze_btree_iterator_non_strict +inline double btree_cost(double my_est) { + return non_strict_cost_of_strict_iterator(my_est, btree_strict_cost(my_est)); +} + +// Non-strict cost of matching in a bitvector. +inline double bitvector_cost() { + return 1.0; +} + +// Strict cost of matching in a bitvector. +// Test used: IteratorBenchmark::analyze_btree_vs_bitvector_iterators_strict +inline double bitvector_strict_cost(double my_est) { + return 1.5 * my_est; } // Strict cost of matching in a disk index posting list. +// Test used: IteratorBenchmark::analyze_term_search_in_disk_index inline double disk_index_strict_cost(double my_est) { return 1.5 * my_est; } +// Non-strict cost of matching in a disk index posting list. +// Test used: IteratorBenchmark::analyze_term_search_in_disk_index +inline double disk_index_cost(double my_est) { + return non_strict_cost_of_strict_iterator(my_est, disk_index_strict_cost(my_est)); +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index b43b560ae2a..bf7f44f0e7a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -34,8 +34,7 @@ HitCollector::sortHitsByDocId() } } -HitCollector::HitCollector(uint32_t numDocs, - uint32_t maxHitsSize) +HitCollector::HitCollector(uint32_t numDocs, uint32_t maxHitsSize) : _numDocs(numDocs), _maxHitsSize(std::min(maxHitsSize, numDocs)), _maxDocIdVectorSize((numDocs + 31) / 32), @@ -63,14 +62,14 @@ HitCollector::RankedHitCollector::collect(uint32_t docId, feature_t score) { HitCollector & hc = this->_hc; if (hc._hits.size() < hc._maxHitsSize) { - if (__builtin_expect(((hc._hits.size() > 0) && + if (__builtin_expect(((!hc._hits.empty()) && (docId < hc._hits.back().first) && (hc._hitsSortOrder == SortOrder::DOC_ID)), false)) { hc._hitsSortOrder = SortOrder::NONE; hc._unordered = true; } - hc._hits.push_back(std::make_pair(docId, score)); + hc._hits.emplace_back(docId, score); } else { collectAndChangeCollector(docId, score); } @@ -86,7 +85,7 @@ HitCollector::BitVectorCollector<CollectRankedHit>::collect(uint32_t docId, feat } void -HitCollector::CollectorBase::replaceHitInVector(uint32_t docId, feature_t score) { +HitCollector::CollectorBase::replaceHitInVector(uint32_t docId, feature_t score) noexcept { // replace lowest scored hit in hit vector std::pop_heap(_hc._hits.begin(), _hc._hits.end(), ScoreComparator()); _hc._hits.back().first = docId; @@ -138,7 +137,7 @@ HitCollector::DocIdCollector<CollectRankedHit>::collect(uint32_t docId, feature_ } HitCollector & hc = this->_hc; if (hc._docIdVector.size() < hc._maxDocIdVectorSize) { - if (__builtin_expect(((hc._docIdVector.size() > 0) && + if (__builtin_expect(((!hc._docIdVector.empty()) && (docId < hc._docIdVector.back()) && (hc._unordered == false)), false)) { @@ -173,7 +172,7 @@ HitCollector::getSortedHitSequence(size_t max_hits) { size_t num_hits = std::min(_hits.size(), max_hits); sortHitsByScore(num_hits); - return SortedHitSequence(_hits.data(), _scoreOrder.data(), num_hits); + return {_hits.data(), _scoreOrder.data(), num_hits}; } void diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index a98a5bdf625..94ffe619bab 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -39,7 +39,7 @@ private: feature_t _adjust; struct ScoreComparator { - bool operator() (const Hit & lhs, const Hit & rhs) const { + bool operator() (const Hit & lhs, const Hit & rhs) const noexcept { if (lhs.second == rhs.second) { return (lhs.first < rhs.first); } @@ -48,7 +48,7 @@ private: }; struct IndirectScoreComparator { - IndirectScoreComparator(const Hit * hits) : _hits(hits) { } + explicit IndirectScoreComparator(const Hit * hits) noexcept : _hits(hits) { } bool operator() (uint32_t lhs, uint32_t rhs) const { if (_hits[lhs].second == _hits[rhs].second) { return (_hits[lhs].first < _hits[rhs].first); @@ -59,17 +59,17 @@ private: }; struct IndirectScoreRadix { - IndirectScoreRadix(const Hit * hits) : _hits(hits) { } - uint64_t operator () (uint32_t v) { + explicit IndirectScoreRadix(const Hit * hits) noexcept : _hits(hits) { } + uint64_t operator () (uint32_t v) const noexcept { return vespalib::convertForSort<double, false>::convert(_hits[v].second); } const Hit * _hits; }; struct DocIdRadix { - uint32_t operator () (const Hit & v) { return v.first; } + uint32_t operator () (const Hit & v) const noexcept { return v.first; } }; struct DocIdComparator { - bool operator() (const Hit & lhs, const Hit & rhs) const { + bool operator() (const Hit & lhs, const Hit & rhs) const noexcept { return (lhs.first < rhs.first); } }; @@ -77,47 +77,47 @@ private: class Collector { public: using UP = std::unique_ptr<Collector>; - virtual ~Collector() {} + virtual ~Collector() = default; virtual void collect(uint32_t docId, feature_t score) = 0; - virtual bool isDocIdCollector() const { return false; } + virtual bool isDocIdCollector() const noexcept { return false; } }; Collector::UP _collector; class CollectorBase : public Collector { public: - CollectorBase(HitCollector &hc) : _hc(hc) { } + explicit CollectorBase(HitCollector &hc) noexcept : _hc(hc) { } void considerForHitVector(uint32_t docId, feature_t score) { if (__builtin_expect((score > _hc._hits[0].second), false)) { replaceHitInVector(docId, score); } } protected: - void replaceHitInVector(uint32_t docId, feature_t score); + VESPA_DLL_LOCAL void replaceHitInVector(uint32_t docId, feature_t score) noexcept; HitCollector &_hc; }; - class RankedHitCollector : public CollectorBase { + class RankedHitCollector final : public CollectorBase { public: - RankedHitCollector(HitCollector &hc) : CollectorBase(hc) { } + explicit RankedHitCollector(HitCollector &hc) noexcept : CollectorBase(hc) { } void collect(uint32_t docId, feature_t score) override; void collectAndChangeCollector(uint32_t docId, feature_t score) __attribute__((noinline)); }; template <bool CollectRankedHit> - class DocIdCollector : public CollectorBase { + class DocIdCollector final : public CollectorBase { public: - DocIdCollector(HitCollector &hc) : CollectorBase(hc) { } + explicit DocIdCollector(HitCollector &hc) noexcept : CollectorBase(hc) { } void collect(uint32_t docId, feature_t score) override; void collectAndChangeCollector(uint32_t docId) __attribute__((noinline)); - bool isDocIdCollector() const override { return true; } + bool isDocIdCollector() const noexcept override { return true; } }; template <bool CollectRankedHit> - class BitVectorCollector : public CollectorBase { + class BitVectorCollector final : public CollectorBase { public: - BitVectorCollector(HitCollector &hc) : CollectorBase(hc) { } - virtual void collect(uint32_t docId, feature_t score) override; + explicit BitVectorCollector(HitCollector &hc) noexcept : CollectorBase(hc) { } + void collect(uint32_t docId, feature_t score) override; }; HitRank getReScore(feature_t score) const { diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index fb1bb0b6cf1..2a0f1de0e44 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -162,10 +162,10 @@ AndNotBlueprint::get_replacement() } void -AndNotBlueprint::sort(Children &children, bool strict, bool sort_by_cost) const +AndNotBlueprint::sort(Children &children, InFlow in_flow) const { - if (sort_by_cost) { - AndNotFlow::sort(children, strict); + if (opt_sort_by_cost()) { + AndNotFlow::sort(children, in_flow.strict()); } else { if (children.size() > 2) { std::sort(children.begin() + 1, children.end(), TieredGreaterEstimate()); @@ -257,10 +257,13 @@ AndBlueprint::get_replacement() } void -AndBlueprint::sort(Children &children, bool strict, bool sort_by_cost) const +AndBlueprint::sort(Children &children, InFlow in_flow) const { - if (sort_by_cost) { - AndFlow::sort(children, strict); + if (opt_sort_by_cost()) { + AndFlow::sort(children, in_flow.strict()); + if (opt_allow_force_strict()) { + AndFlow::reorder_for_extra_strictness(children, in_flow, 3); + } } else { std::sort(children.begin(), children.end(), TieredLessEstimate()); } @@ -357,10 +360,10 @@ OrBlueprint::get_replacement() } void -OrBlueprint::sort(Children &children, bool strict, bool sort_by_cost) const +OrBlueprint::sort(Children &children, InFlow in_flow) const { - if (sort_by_cost) { - OrFlow::sort(children, strict); + if (opt_sort_by_cost()) { + OrFlow::sort(children, in_flow.strict()); } else { std::sort(children.begin(), children.end(), TieredGreaterEstimate()); } @@ -445,8 +448,17 @@ WeakAndBlueprint::exposeFields() const return {}; } +Blueprint::UP +WeakAndBlueprint::get_replacement() +{ + if (childCnt() == 1) { + return removeChild(0); + } + return {}; +} + void -WeakAndBlueprint::sort(Children &, bool, bool) const +WeakAndBlueprint::sort(Children &, InFlow) const { // order needs to stay the same as _weights } @@ -508,10 +520,10 @@ NearBlueprint::exposeFields() const } void -NearBlueprint::sort(Children &children, bool strict, bool sort_by_cost) const +NearBlueprint::sort(Children &children, InFlow in_flow) const { - if (sort_by_cost) { - AndFlow::sort(children, strict); + if (opt_sort_by_cost()) { + AndFlow::sort(children, in_flow.strict()); } else { std::sort(children.begin(), children.end(), TieredLessEstimate()); } @@ -573,7 +585,7 @@ ONearBlueprint::exposeFields() const } void -ONearBlueprint::sort(Children &, bool, bool) const +ONearBlueprint::sort(Children &, InFlow) const { // ordered near cannot sort children here } @@ -659,7 +671,7 @@ RankBlueprint::get_replacement() } void -RankBlueprint::sort(Children &, bool, bool) const +RankBlueprint::sort(Children &, InFlow) const { } @@ -740,7 +752,7 @@ SourceBlenderBlueprint::exposeFields() const } void -SourceBlenderBlueprint::sort(Children &, bool, bool) const +SourceBlenderBlueprint::sort(Children &, InFlow) const { } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index 5d6d098510f..dfed40f4d1b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -21,7 +21,7 @@ public: void optimize_self(OptimizePass pass) override; AndNotBlueprint * asAndNot() noexcept final { return this; } Blueprint::UP get_replacement() override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, fef::MatchData &md) const override; @@ -48,7 +48,7 @@ public: void optimize_self(OptimizePass pass) override; AndBlueprint * asAnd() noexcept final { return this; } Blueprint::UP get_replacement() override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, fef::MatchData &md) const override; @@ -72,7 +72,7 @@ public: void optimize_self(OptimizePass pass) override; OrBlueprint * asOr() noexcept final { return this; } Blueprint::UP get_replacement() override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, fef::MatchData &md) const override; @@ -96,7 +96,8 @@ public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void sort(Children &children, bool strict, bool sort_on_cost) const override; + Blueprint::UP get_replacement() override; + void sort(Children &children, InFlow in_flow) const override; bool always_needs_unpack() const override; WeakAndBlueprint * asWeakAnd() noexcept final { return this; } SearchIterator::UP @@ -126,7 +127,7 @@ public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIteratorUP createSearch(fef::MatchData &md) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, @@ -148,7 +149,7 @@ public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIteratorUP createSearch(fef::MatchData &md) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, @@ -168,7 +169,7 @@ public: FieldSpecBaseList exposeFields() const override; void optimize_self(OptimizePass pass) override; Blueprint::UP get_replacement() override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; bool isRank() const noexcept final { return true; } SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, @@ -196,7 +197,7 @@ public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void sort(Children &children, bool strict, bool sort_by_cost) const override; + void sort(Children &children, InFlow in_flow) const override; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, fef::MatchData &md) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index d28f6077905..c76fe3363e4 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -19,17 +19,17 @@ namespace search::queryeval { * Keeps a heap of the K best hit distances. * Currently always does brute-force scanning, which is very expensive. **/ -template <bool strict, bool has_filter> -class NearestNeighborImpl : public NearestNeighborIterator +template <bool strict, bool has_filter, bool has_single_subspace> +class NearestNeighborImpl final : public NearestNeighborIterator { public: - NearestNeighborImpl(Params params_in) + explicit NearestNeighborImpl(Params params_in) : NearestNeighborIterator(std::move(params_in)), _lastScore(0.0) { } - ~NearestNeighborImpl(); + ~NearestNeighborImpl() override; void doSeek(uint32_t docId) override { double distanceLimit = params().distanceHeap.distanceLimit(); @@ -61,39 +61,47 @@ public: private: double computeDistance(uint32_t docId, double limit) { - return params().distance_calc->calc_with_limit(docId, limit); + return params().distance_calc->template calc_with_limit<has_single_subspace>(docId, limit); } double _lastScore; }; -template <bool strict, bool has_filter> -NearestNeighborImpl<strict, has_filter>::~NearestNeighborImpl() = default; +template <bool strict, bool has_filter, bool has_single_subspace> +NearestNeighborImpl<strict, has_filter, has_single_subspace>::~NearestNeighborImpl() = default; namespace { +template <bool strict, bool has_filter> +std::unique_ptr<NearestNeighborIterator> +resolve_single_subspace(NearestNeighborIterator::Params params) +{ + if (params.distance_calc->has_single_subspace()) { + using NNI = NearestNeighborImpl<strict, has_filter, true>; + return std::make_unique<NNI>(std::move(params)); + } else { + using NNI = NearestNeighborImpl<strict, has_filter, false>; + return std::make_unique<NNI>(std::move(params)); + } +} + template <bool has_filter> std::unique_ptr<NearestNeighborIterator> resolve_strict(bool strict, NearestNeighborIterator::Params params) { if (strict) { - using NNI = NearestNeighborImpl<true, has_filter>; - return std::make_unique<NNI>(std::move(params)); + return resolve_single_subspace<true, has_filter>(std::move(params)); } else { - using NNI = NearestNeighborImpl<false, has_filter>; - return std::make_unique<NNI>(std::move(params)); + return resolve_single_subspace<false, has_filter>(std::move(params)); } } } // namespace <unnamed> std::unique_ptr<NearestNeighborIterator> -NearestNeighborIterator::create( - bool strict, - fef::TermFieldMatchData &tfmd, - std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, - NearestNeighborDistanceHeap &distanceHeap, - const GlobalFilter &filter) +NearestNeighborIterator::create(bool strict, fef::TermFieldMatchData &tfmd, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, + NearestNeighborDistanceHeap &distanceHeap, const GlobalFilter &filter) { Params params(tfmd, std::move(distance_calc), distanceHeap, filter); if (filter.is_active()) { diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index b34c9df47b9..177c732a44d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -39,7 +39,7 @@ public: {} }; - NearestNeighborIterator(Params params_in) + explicit NearestNeighborIterator(Params params_in) : _params(std::move(params_in)) {} diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index f5231cf3509..f915a7df335 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -7,12 +7,11 @@ #include <vespa/vespalib/util/priority_queue.h> #include <vespa/vespalib/objects/visit.hpp> -namespace search { -namespace queryeval { +namespace search::queryeval { namespace wand { template <typename FutureHeap, typename PastHeap, bool IS_STRICT> -class WeakAndSearchLR : public WeakAndSearch +class WeakAndSearchLR final : public WeakAndSearch { private: using Scores = vespalib::PriorityQueue<score_t>; @@ -47,7 +46,7 @@ public: : _terms(terms, TermFrequencyScorer(), 0, - fef::MatchData::UP(nullptr)), + fef::MatchData::UP()), _heaps(DocIdOrder(_terms.docId()), _terms.size()), _algo(), _threshold(1), @@ -55,9 +54,9 @@ public: _n(n) { } - virtual size_t get_num_terms() const override { return _terms.size(); } - virtual int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } - virtual score_t get_max_score(size_t idx) const override { return _terms.maxScore(idx); } + size_t get_num_terms() const override { return _terms.size(); } + int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } + score_t get_max_score(size_t idx) const override { return _terms.maxScore(idx); } const Terms &getTerms() const override { return _terms.input_terms(); } uint32_t getN() const override { return _n; } void doSeek(uint32_t docid) override { @@ -138,5 +137,4 @@ WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) //----------------------------------------------------------------------------- -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 83a1968f4a1..e34c5791ede 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -24,6 +24,7 @@ vespa_add_library(searchlib_tensor OBJECT hnsw_multi_best_neighbors.cpp hnsw_nodeid_mapping.cpp hnsw_single_best_neighbors.cpp + hnsw_test_node.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp inv_log_level_generator.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index 14953011e22..af99260979d 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -2,7 +2,9 @@ #include "angular_distance.h" #include "temporary_vector_store.h" +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <numbers> +#include <cmath> using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; @@ -10,47 +12,15 @@ using vespalib::eval::TypedCells; namespace search::tensor { -namespace { - -struct CalcAngular { - template <typename LCT, typename RCT> - static double invoke(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) - { - auto lhs_vector = lhs.unsafe_typify<LCT>(); - auto rhs_vector = rhs.unsafe_typify<RCT>(); - - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - double a_norm_sq = 0.0; - double b_norm_sq = 0.0; - double dot_product = 0.0; - for (size_t i = 0; i < sz; ++i) { - double a = lhs_vector[i]; - double b = rhs_vector[i]; - a_norm_sq += a*a; - b_norm_sq += b*b; - dot_product += a*b; - } - double squared_norms = a_norm_sq * b_norm_sq; - double div = (squared_norms > 0) ? sqrt(squared_norms) : 1.0; - double cosine_similarity = dot_product / div; - double distance = 1.0 - cosine_similarity; // in range [0,2] - return std::max(0.0, distance); - } -}; - -} - template<typename FloatType> -class BoundAngularDistance : public BoundDistanceFunction { +class BoundAngularDistance final : public BoundDistanceFunction { private: const vespalib::hwaccelrated::IAccelrated & _computer; mutable TemporaryVectorStore<FloatType> _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs; double _lhs_norm_sq; public: - BoundAngularDistance(const vespalib::eval::TypedCells& lhs) + explicit BoundAngularDistance(TypedCells lhs) : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) @@ -58,10 +28,9 @@ public: auto a = _lhs.data(); _lhs_norm_sq = _computer.dotProduct(a, a, lhs.size); } - double calc(const vespalib::eval::TypedCells& rhs) const override { + double calc(TypedCells rhs) const noexcept override { size_t sz = _lhs.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs.data(); auto b = rhs_vector.data(); double b_norm_sq = _computer.dotProduct(b, b, sz); @@ -72,7 +41,7 @@ public: double distance = 1.0 - cosine_similarity; // in range [0,2] return distance; } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { if (threshold < 0.0) { return 0.0; } @@ -82,7 +51,7 @@ public: double cosine_similarity = cos(threshold); return 1.0 - cosine_similarity; } - double to_rawscore(double distance) const override { + double to_rawscore(double distance) const noexcept override { double cosine_similarity = 1.0 - distance; // should be in the range [-1,1] but roundoff may cause problems: cosine_similarity = std::min(1.0, cosine_similarity); @@ -91,7 +60,7 @@ public: double score = 1.0 / (1.0 + angle_distance); return score; } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + double calc_with_limit(TypedCells rhs, double) const noexcept override { return calc(rhs); } }; @@ -101,14 +70,14 @@ template class BoundAngularDistance<double>; template <typename FloatType> BoundDistanceFunction::UP -AngularDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { +AngularDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) { using DFT = BoundAngularDistance<FloatType>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP -AngularDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +AngularDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) { using DFT = BoundAngularDistance<FloatType>; return std::make_unique<DFT>(lhs); } diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.h b/searchlib/src/vespa/searchlib/tensor/angular_distance.h index f5e8589fe6a..5e0a060e060 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.h @@ -2,12 +2,7 @@ #pragma once -#include "distance_function.h" -#include "bound_distance_function.h" #include "distance_function_factory.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <cmath> namespace search::tensor { @@ -20,8 +15,8 @@ template <typename FloatType> class AngularDistanceFunctionFactory : public DistanceFunctionFactory { public: AngularDistanceFunctionFactory() = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h index c89619d9a77..85089196a7a 100644 --- a/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/bound_distance_function.h @@ -2,13 +2,9 @@ #pragma once -#include <memory> -#include <vespa/eval/eval/cell_type.h> -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/util/arrayref.h> #include "distance_function.h" - -namespace vespalib::eval { struct TypedCells; } +#include <vespa/eval/eval/typed_cells.h> +#include <memory> namespace search::tensor { @@ -22,17 +18,17 @@ namespace search::tensor { class BoundDistanceFunction : public DistanceConverter { public: using UP = std::unique_ptr<BoundDistanceFunction>; + using TypedCells = vespalib::eval::TypedCells; - BoundDistanceFunction() = default; + BoundDistanceFunction() noexcept = default; - virtual ~BoundDistanceFunction() = default; + ~BoundDistanceFunction() override = default; // calculate internal distance (comparable) - virtual double calc(const vespalib::eval::TypedCells& rhs) const = 0; + virtual double calc(TypedCells rhs) const noexcept = 0; // calculate internal distance, early return allowed if > limit - virtual double calc_with_limit(const vespalib::eval::TypedCells& rhs, - double limit) const = 0; + virtual double calc_with_limit(TypedCells rhs, double limit) const noexcept = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index fb74dd51fa3..0dbb9c34010 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -30,14 +30,14 @@ DenseTensorAttribute::extract_cells_ref(DocId docId) const } vespalib::eval::TypedCells -DenseTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const +DenseTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const noexcept { EntryRef ref = (subspace == 0) ? acquire_entry_ref(docid) : EntryRef(); return _denseTensorStore.get_typed_cells(ref); } VectorBundle -DenseTensorAttribute::get_vectors(uint32_t docid) const +DenseTensorAttribute::get_vectors(uint32_t docid) const noexcept { EntryRef ref = acquire_entry_ref(docid); return _denseTensorStore.get_vectors(ref); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 03c976bd6b3..c07bfcc358e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -26,8 +26,8 @@ public: bool supports_extract_cells_ref() const override { return true; } // Implements DocVectorAccess - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; - VectorBundle get_vectors(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override; + VectorBundle get_vectors(uint32_t docid) const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp index 12dd6aa2bca..cf0e9adc095 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.cpp @@ -74,7 +74,7 @@ DirectTensorAttribute::get_tensor_ref(DocId docId) const } vespalib::eval::TypedCells -DirectTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const +DirectTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const noexcept { EntryRef ref = acquire_entry_ref(docid); auto vectors = _direct_store.get_vectors(ref); @@ -82,7 +82,7 @@ DirectTensorAttribute::get_vector(uint32_t docid, uint32_t subspace) const } VectorBundle -DirectTensorAttribute::get_vectors(uint32_t docid) const +DirectTensorAttribute::get_vectors(uint32_t docid) const noexcept { EntryRef ref = acquire_entry_ref(docid); return _direct_store.get_vectors(ref); diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h index a4f673ea99f..64f62650615 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_attribute.h @@ -26,8 +26,8 @@ public: bool supports_get_tensor_ref() const override { return true; } // Implements DocVectorAccess - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; - VectorBundle get_vectors(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override; + VectorBundle get_vectors(uint32_t docid) const noexcept override; }; } // namespace search::tensor diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index 44bbbba65d6..6edb654d5bf 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -42,11 +42,11 @@ private: EntryRef add_entry(TensorSP tensor); public: - DirectTensorStore(const vespalib::eval::ValueType& tensor_type); + explicit DirectTensorStore(const vespalib::eval::ValueType& tensor_type); ~DirectTensorStore() override; using RefType = TensorStoreType::RefType; - const vespalib::eval::Value * get_tensor_ptr(EntryRef ref) const { + const vespalib::eval::Value * get_tensor_ptr(EntryRef ref) const noexcept { if (!ref.valid()) { return nullptr; } @@ -65,12 +65,12 @@ public: vespalib::eval::TypedCells get_empty_subspace() const noexcept { return _empty.cells(); } - VectorBundle get_vectors(EntryRef ref) const { + VectorBundle get_vectors(EntryRef ref) const noexcept { auto tensor = get_tensor_ptr(ref); if (tensor == nullptr) { - return VectorBundle(); + return {}; } - return VectorBundle(tensor->cells().data, tensor->index().size(), _subspace_type); + return {tensor->cells().data, static_cast<uint32_t>(tensor->index().size()), _subspace_type}; } }; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index eab75537071..7ff9448c5af 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -5,6 +5,7 @@ #include "distance_function_factory.h" #include "i_tensor_attribute.h" #include "vector_bundle.h" +#include <vespa/eval/eval/value_type.h> #include <optional> namespace vespalib::eval { struct Value; } @@ -32,34 +33,55 @@ public: ~DistanceCalculator(); const tensor::ITensorAttribute& attribute_tensor() const { return _attr_tensor; } - const vespalib::eval::Value& query_tensor() const { + const vespalib::eval::Value& query_tensor() const noexcept{ assert(_query_tensor != nullptr); return *_query_tensor; } - const BoundDistanceFunction& function() const { return *_dist_fun; } + const BoundDistanceFunction& function() const noexcept { return *_dist_fun; } + bool has_single_subspace() const noexcept { return _attr_tensor.getTensorType().is_dense(); } - double calc_raw_score(uint32_t docid) const { - auto vectors = _attr_tensor.get_vectors(docid); - double result = _dist_fun->min_rawscore(); - for (uint32_t i = 0; i < vectors.subspaces(); ++i) { - double distance = _dist_fun->calc(vectors.cells(i)); - double score = _dist_fun->to_rawscore(distance); - result = std::max(result, score); + template<bool has_single_subspace> + double calc_raw_score(uint32_t docid) const noexcept { + if (has_single_subspace) { + auto cells = _attr_tensor.get_vector(docid, 0); + double min_rawscore = _dist_fun->min_rawscore(); + if ( ! cells.valid() ) [[unlikely]] { + return min_rawscore; + } + return std::max(min_rawscore, _dist_fun->to_rawscore(_dist_fun->calc(cells))); + } else { + auto vectors = _attr_tensor.get_vectors(docid); + double result = _dist_fun->min_rawscore(); + for (uint32_t i = 0; i < vectors.subspaces(); ++i) { + double distance = _dist_fun->calc(vectors.cells(i)); + double score = _dist_fun->to_rawscore(distance); + result = std::max(result, score); + } + return result; } - return result; + } - double calc_with_limit(uint32_t docid, double limit) const { - auto vectors = _attr_tensor.get_vectors(docid); - double result = std::numeric_limits<double>::max(); - for (uint32_t i = 0; i < vectors.subspaces(); ++i) { - double distance = _dist_fun->calc_with_limit(vectors.cells(i), limit); - result = std::min(result, distance); + template<bool has_single_subspace> + double calc_with_limit(uint32_t docid, double limit) const noexcept { + if (has_single_subspace) { + auto cells = _attr_tensor.get_vector(docid, 0); + if ( ! cells.valid() ) [[unlikely]] { + return std::numeric_limits<double>::max(); + } + return _dist_fun->calc_with_limit(cells, limit); + } else { + auto vectors = _attr_tensor.get_vectors(docid); + double result = std::numeric_limits<double>::max(); + for (uint32_t i = 0; i < vectors.subspaces(); ++i) { + double distance = _dist_fun->calc_with_limit(vectors.cells(i), limit); + result = std::min(result, distance); + } + return result; } - return result; } - void calc_closest_subspace(VectorBundle vectors, std::optional<uint32_t>& closest_subspace, double& best_distance) { + void calc_closest_subspace(VectorBundle vectors, std::optional<uint32_t>& closest_subspace, double& best_distance) noexcept { for (uint32_t i = 0; i < vectors.subspaces(); ++i) { double distance = _dist_fun->calc(vectors.cells(i)); if (!closest_subspace.has_value() || distance < best_distance) { @@ -69,7 +91,7 @@ public: } } - std::optional<uint32_t> calc_closest_subspace(VectorBundle vectors) { + std::optional<uint32_t> calc_closest_subspace(VectorBundle vectors) noexcept { double best_distance = 0.0; std::optional<uint32_t> closest_subspace; calc_closest_subspace(vectors, closest_subspace, best_distance); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index c2e8305038c..9a2db8dfac0 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -2,11 +2,6 @@ #pragma once -#include <memory> -#include <vespa/eval/eval/cell_type.h> - -namespace vespalib::eval { struct TypedCells; } - namespace search::tensor { class DistanceConverter { @@ -16,25 +11,25 @@ public: /** * Convert threshold (external distance units) to internal units. */ - virtual double convert_threshold(double threshold) const = 0; + virtual double convert_threshold(double threshold) const noexcept = 0; /** * Convert internal distance to rawscore (also used as closeness). */ - virtual double to_rawscore(double distance) const = 0; + virtual double to_rawscore(double distance) const noexcept = 0; /** * Convert rawscore to external distance. * Override this when the rawscore is NOT defined as (1.0 / (1.0 + external_distance)). */ - virtual double to_distance(double rawscore) const { + virtual double to_distance(double rawscore) const noexcept { return (1.0 / rawscore) - 1.0; } /** * The minimum rawscore (also used as closeness) that this distance function can return. */ - virtual double min_rawscore() const { + virtual double min_rawscore() const noexcept { return 0.0; } }; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index 4749a8549a6..ed08df5866e 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -3,22 +3,14 @@ #include "distance_function_factory.h" #include "distance_functions.h" #include "mips_distance_transform.h" -#include <vespa/vespalib/util/typify.h> -#include <vespa/vespalib/util/array.h> -#include <vespa/vespalib/util/arrayref.h> -#include <vespa/log/log.h> - -LOG_SETUP(".searchlib.tensor.distance_function_factory"); using search::attribute::DistanceMetric; using vespalib::eval::CellType; -using vespalib::eval::ValueType; namespace search::tensor { std::unique_ptr<DistanceFunctionFactory> -make_distance_function_factory(search::attribute::DistanceMetric variant, - vespalib::eval::CellType cell_type) +make_distance_function_factory(DistanceMetric variant, CellType cell_type) { switch (variant) { case DistanceMetric::Angular: diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h index 829ed7fae13..356366d6a77 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h @@ -4,7 +4,6 @@ #include "distance_function.h" #include "bound_distance_function.h" -#include <vespa/eval/eval/value_type.h> #include <vespa/searchcommon/attribute/distance_metric.h> namespace search::tensor { @@ -15,10 +14,11 @@ namespace search::tensor { * for one particular vector in the distance function object. */ struct DistanceFunctionFactory { - DistanceFunctionFactory() = default; - virtual ~DistanceFunctionFactory() {} - virtual BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) = 0; - virtual BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) = 0; + using TypedCells = vespalib::eval::TypedCells; + DistanceFunctionFactory() noexcept = default; + virtual ~DistanceFunctionFactory() = default; + virtual BoundDistanceFunction::UP for_query_vector(TypedCells lhs) = 0; + virtual BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) = 0; using UP = std::unique_ptr<DistanceFunctionFactory>; }; diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h index 477d5e1dc8a..dd68171dd59 100644 --- a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h +++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h @@ -16,9 +16,9 @@ class VectorBundle; */ class DocVectorAccess { public: - virtual ~DocVectorAccess() {} - virtual vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const = 0; - virtual VectorBundle get_vectors(uint32_t docid) const = 0; + virtual ~DocVectorAccess() = default; + virtual vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept = 0; + virtual VectorBundle get_vectors(uint32_t docid) const noexcept = 0; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp index cfc420d9ecd..a7168b5eae6 100644 --- a/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp +++ b/searchlib/src/vespa/searchlib/tensor/empty_subspace.cpp @@ -10,7 +10,8 @@ EmptySubspace::EmptySubspace(const SubspaceType& type) _cells() { _empty_space.resize(type.mem_size()); - _cells = vespalib::eval::TypedCells(&_empty_space[0], type.cell_type(), type.size()); + // Set size to zero to signal empty/invalid subspace + _cells = vespalib::eval::TypedCells(_empty_space.data(), type.cell_type(), 0); } EmptySubspace::~EmptySubspace() = default; diff --git a/searchlib/src/vespa/searchlib/tensor/empty_subspace.h b/searchlib/src/vespa/searchlib/tensor/empty_subspace.h index dd0ab9264c4..4043ec122e6 100644 --- a/searchlib/src/vespa/searchlib/tensor/empty_subspace.h +++ b/searchlib/src/vespa/searchlib/tensor/empty_subspace.h @@ -10,7 +10,7 @@ namespace search::tensor { class SubspaceType; /* - * Class containg an empty subspace, used as a bad fallback when we cannot + * Class containing an empty subspace, used as a bad fallback when we cannot * get a real subspace. */ class EmptySubspace diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index 3efc8c3a5ea..3ab3a1123eb 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -2,39 +2,20 @@ #include "euclidean_distance.h" #include "temporary_vector_store.h" +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <cmath> using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; +using vespalib::eval::TypedCells; namespace search::tensor { -namespace { - -struct CalcEuclidean { - template <typename LCT, typename RCT> - static double invoke(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) - { - auto lhs_vector = lhs.unsafe_typify<LCT>(); - auto rhs_vector = rhs.unsafe_typify<RCT>(); - double sum = 0.0; - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - for (size_t i = 0; i < sz; ++i) { - double diff = lhs_vector[i] - rhs_vector[i]; - sum += diff*diff; - } - return sum; - } -}; - -} - using vespalib::eval::Int8Float; using vespalib::BFloat16; template<typename AttributeCellType> -class BoundEuclideanDistance : public BoundDistanceFunction { +class BoundEuclideanDistance final : public BoundDistanceFunction { using FloatType = std::conditional_t<std::is_same<AttributeCellType,BFloat16>::value,float,AttributeCellType>; private: const vespalib::hwaccelrated::IAccelrated & _computer; @@ -44,28 +25,26 @@ private: static const float *cast(const float * p) { return p; } static const int8_t *cast(const Int8Float * p) { return reinterpret_cast<const int8_t *>(p); } public: - BoundEuclideanDistance(const vespalib::eval::TypedCells& lhs) + explicit BoundEuclideanDistance(TypedCells lhs) : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), _tmpSpace(lhs.size), _lhs_vector(_tmpSpace.storeLhs(lhs)) {} - double calc(const vespalib::eval::TypedCells& rhs) const override { - size_t sz = _lhs_vector.size(); + double calc(TypedCells rhs) const noexcept override { vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs_vector.data(); auto b = rhs_vector.data(); - return _computer.squaredEuclideanDistance(cast(a), cast(b), sz); + return _computer.squaredEuclideanDistance(cast(a), cast(b), _lhs_vector.size()); } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { return threshold*threshold; } - double to_rawscore(double distance) const override { + double to_rawscore(double distance) const noexcept override { double d = sqrt(distance); double score = 1.0 / (1.0 + d); return score; } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double limit) const override { + double calc_with_limit(TypedCells rhs, double limit) const noexcept override { vespalib::ConstArrayRef<AttributeCellType> rhs_vector = rhs.typify<AttributeCellType>(); double sum = 0.0; size_t sz = _lhs_vector.size(); @@ -85,14 +64,14 @@ template class BoundEuclideanDistance<double>; template <typename FloatType> BoundDistanceFunction::UP -EuclideanDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { +EuclideanDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) { using DFT = BoundEuclideanDistance<FloatType>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP -EuclideanDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +EuclideanDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) { using DFT = BoundEuclideanDistance<FloatType>; return std::make_unique<DFT>(lhs); } diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h index 42097f8b39b..8c39a12bf86 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h @@ -2,11 +2,7 @@ #pragma once -#include "distance_function.h" #include "distance_function_factory.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <cmath> namespace search::tensor { @@ -18,9 +14,9 @@ namespace search::tensor { template <typename FloatType> class EuclideanDistanceFunctionFactory : public DistanceFunctionFactory { public: - EuclideanDistanceFunctionFactory() = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + EuclideanDistanceFunctionFactory() noexcept = default; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp index 7b6c40c643e..f5484f40271 100644 --- a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp @@ -3,6 +3,7 @@ #include "geo_degrees_distance.h" #include "temporary_vector_store.h" #include <numbers> +#include <cmath> using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; @@ -15,7 +16,7 @@ namespace search::tensor { * Uses the haversine formula directly from: * https://en.wikipedia.org/wiki/Haversine_formula **/ -class BoundGeoDistance : public BoundDistanceFunction { +class BoundGeoDistance final : public BoundDistanceFunction { private: mutable TemporaryVectorStore<double> _tmpSpace; const vespalib::ConstArrayRef<double> _lh_vector; @@ -26,16 +27,16 @@ public: static constexpr double degrees_to_radians = M_PI / 180.0; // haversine function: - static double haversine(double angle) { + static double haversine(double angle) noexcept { double s = sin(0.5*angle); return s*s; } - BoundGeoDistance(const vespalib::eval::TypedCells& lhs) + explicit BoundGeoDistance(TypedCells lhs) : _tmpSpace(lhs.size), _lh_vector(_tmpSpace.storeLhs(lhs)) {} - double calc(const vespalib::eval::TypedCells& rhs) const override { + double calc(TypedCells rhs) const noexcept override { vespalib::ConstArrayRef<double> rhs_vector = _tmpSpace.convertRhs(rhs); assert(2 == _lh_vector.size()); assert(2 == rhs_vector.size()); @@ -56,7 +57,7 @@ public: double hav_central_angle = hav_lat + cos(lat_A)*cos(lat_B)*hav_lon; return hav_central_angle; } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { if (threshold < 0.0) { return 0.0; } @@ -68,25 +69,25 @@ public: double rt_hav = sin(half_angle); return rt_hav * rt_hav; } - double to_rawscore(double distance) const override { + double to_rawscore(double distance) const noexcept override { double hav_diff = sqrt(distance); // distance in kilometers: double d = 2 * asin(hav_diff) * earth_mean_radius; // km to rawscore: return 1.0 / (1.0 + d); } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + double calc_with_limit(TypedCells rhs, double) const noexcept override { return calc(rhs); } }; BoundDistanceFunction::UP -GeoDistanceFunctionFactory::for_query_vector(const vespalib::eval::TypedCells& lhs) { +GeoDistanceFunctionFactory::for_query_vector(TypedCells lhs) { return std::make_unique<BoundGeoDistance>(lhs); } BoundDistanceFunction::UP -GeoDistanceFunctionFactory::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +GeoDistanceFunctionFactory::for_insertion_vector(TypedCells lhs) { return std::make_unique<BoundGeoDistance>(lhs); } diff --git a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h index f1af976b91f..1464898421b 100644 --- a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h @@ -2,12 +2,7 @@ #pragma once -#include "distance_function.h" #include "distance_function_factory.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <vespa/vespalib/util/typify.h> -#include <cmath> namespace search::tensor { @@ -19,8 +14,8 @@ namespace search::tensor { class GeoDistanceFunctionFactory : public DistanceFunctionFactory { public: GeoDistanceFunctionFactory() = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp index a1dc8cc52f7..7f29a100492 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp @@ -6,50 +6,27 @@ using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; +using vespalib::eval::TypedCells; namespace search::tensor { -namespace { - -struct CalcHamming { - template <typename LCT, typename RCT> - static double invoke(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) - { - auto lhs_vector = lhs.unsafe_typify<LCT>(); - auto rhs_vector = rhs.unsafe_typify<RCT>(); - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - size_t sum = 0; - for (size_t i = 0; i < sz; ++i) { - sum += (lhs_vector[i] == rhs_vector[i]) ? 0 : 1; - } - return (double)sum; - } -}; - -} - using vespalib::eval::Int8Float; template<typename FloatType> -class BoundHammingDistance : public BoundDistanceFunction { +class BoundHammingDistance final : public BoundDistanceFunction { private: mutable TemporaryVectorStore<FloatType> _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs_vector; public: - BoundHammingDistance(const vespalib::eval::TypedCells& lhs) + explicit BoundHammingDistance(TypedCells lhs) : _tmpSpace(lhs.size), _lhs_vector(_tmpSpace.storeLhs(lhs)) {} - double calc(const vespalib::eval::TypedCells& rhs) const override { + double calc(TypedCells rhs) const noexcept override { size_t sz = _lhs_vector.size(); vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); - auto a = _lhs_vector.data(); - auto b = rhs_vector.data(); if constexpr (std::is_same<Int8Float, FloatType>::value) { - return (double) vespalib::binary_hamming_distance(a, b, sz); + return (double) vespalib::binary_hamming_distance(_lhs_vector.data(), rhs_vector.data(), sz); } else { size_t sum = 0; for (size_t i = 0; i < sz; ++i) { @@ -58,14 +35,13 @@ public: return (double)sum; } } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { return threshold; } - double to_rawscore(double distance) const override { - double score = 1.0 / (1.0 + distance); - return score; + double to_rawscore(double distance) const noexcept override { + return 1.0 / (1.0 + distance); } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + double calc_with_limit(TypedCells rhs, double) const noexcept override { // consider optimizing: return calc(rhs); } @@ -73,14 +49,14 @@ public: template <typename FloatType> BoundDistanceFunction::UP -HammingDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { +HammingDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) { using DFT = BoundHammingDistance<FloatType>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP -HammingDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +HammingDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) { using DFT = BoundHammingDistance<FloatType>; return std::make_unique<DFT>(lhs); } diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h index 32e2be99214..6e7f96e1e2f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h @@ -2,11 +2,7 @@ #pragma once -#include "distance_function.h" #include "distance_function_factory.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/util/typify.h> -#include <cmath> namespace search::tensor { @@ -20,8 +16,8 @@ template <typename FloatType> class HammingDistanceFunctionFactory : public DistanceFunctionFactory { public: HammingDistanceFunctionFactory() = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index bf62fcdaa23..cdc3e81782a 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -2,7 +2,6 @@ #include "hnsw_index.h" #include "bitvector_visited_tracker.h" -#include "distance_function.h" #include "hash_set_visited_tracker.h" #include "hnsw_index_loader.hpp" #include "hnsw_index_saver.h" @@ -145,6 +144,9 @@ PreparedAddDoc::~PreparedAddDoc() = default; PreparedAddDoc::PreparedAddDoc(PreparedAddDoc&& other) noexcept = default; } +SelectResult::SelectResult() noexcept = default; +SelectResult::~SelectResult() = default; + template <HnswIndexType type> ArrayStoreConfig HnswIndex<type>::make_default_level_array_store_config() @@ -180,7 +182,16 @@ template <HnswIndexType type> bool HnswIndex<type>::have_closer_distance(HnswTraversalCandidate candidate, const HnswTraversalCandidateVector& result) const { - auto df = _distance_ff->for_insertion_vector(get_vector(candidate.nodeid)); + auto candidate_vector = get_vector(candidate.nodeid); + if (!candidate_vector.valid()) { + /* + * We are in a read thread and the write thread has removed the + * tensor for the candidate. Return true to prevent the candidate + * from being considered. + */ + return true; + } + auto df = _distance_ff->for_insertion_vector(candidate_vector); for (const auto & neighbor : result) { double dist = calc_distance(*df, neighbor.nodeid); if (dist < candidate.distance) { @@ -192,7 +203,7 @@ HnswIndex<type>::have_closer_distance(HnswTraversalCandidate candidate, const Hn template <HnswIndexType type> template <typename HnswCandidateVectorT> -typename HnswIndex<type>::SelectResult +SelectResult HnswIndex<type>::select_neighbors_simple(const HnswCandidateVectorT& neighbors, uint32_t max_links) const { HnswCandidateVectorT sorted(neighbors); @@ -210,7 +221,7 @@ HnswIndex<type>::select_neighbors_simple(const HnswCandidateVectorT& neighbors, template <HnswIndexType type> template <typename HnswCandidateVectorT> -typename HnswIndex<type>::SelectResult +SelectResult HnswIndex<type>::select_neighbors_heuristic(const HnswCandidateVectorT& neighbors, uint32_t max_links) const { SelectResult result; @@ -239,7 +250,7 @@ HnswIndex<type>::select_neighbors_heuristic(const HnswCandidateVectorT& neighbor template <HnswIndexType type> template <typename HnswCandidateVectorT> -typename HnswIndex<type>::SelectResult +SelectResult HnswIndex<type>::select_neighbors(const HnswCandidateVectorT& neighbors, uint32_t max_links) const { if (_cfg.heuristic_select_neighbors()) { @@ -303,12 +314,29 @@ HnswIndex<type>::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32 _graph.set_link_array(remove_from, level, new_links); } +namespace { + +double +calc_distance_helper(const BoundDistanceFunction &df, vespalib::eval::TypedCells rhs) +{ + if (!rhs.valid()) [[unlikely]] { + /* + * We are in a read thread and the write thread has removed the + * tensor. + */ + return std::numeric_limits<double>::max(); + } + return df.calc(rhs); +} + +} + template <HnswIndexType type> double HnswIndex<type>::calc_distance(const BoundDistanceFunction &df, uint32_t rhs_nodeid) const { auto rhs = get_vector(rhs_nodeid); - return df.calc(rhs); + return calc_distance_helper(df, rhs); } template <HnswIndexType type> @@ -316,7 +344,7 @@ double HnswIndex<type>::calc_distance(const BoundDistanceFunction &df, uint32_t rhs_docid, uint32_t rhs_subspace) const { auto rhs = get_vector(rhs_docid, rhs_subspace); - return df.calc(rhs); + return calc_distance_helper(df, rhs); } template <HnswIndexType type> @@ -556,7 +584,7 @@ HnswIndex<type>::internal_prepare_add_node(PreparedAddDoc& op, TypedCells input_ } template <HnswIndexType type> -typename HnswIndex<type>::LinkArray +LinkArray HnswIndex<type>::filter_valid_nodeids(uint32_t level, const typename PreparedAddNode::Links &neighbors, uint32_t self_nodeid) { LinkArray valid; @@ -969,7 +997,7 @@ HnswIndex<type>::get_node(uint32_t nodeid) const { auto levels_ref = _graph.acquire_levels_ref(nodeid); if (!levels_ref.valid()) { - return HnswTestNode(); + return {}; } auto levels = _graph.levels_store.get(levels_ref); HnswTestNode::LevelArray result; @@ -979,7 +1007,7 @@ HnswIndex<type>::get_node(uint32_t nodeid) const std::sort(result_links.begin(), result_links.end()); result.push_back(result_links); } - return HnswTestNode(result); + return {std::move(result)}; } template <HnswIndexType type> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index d94b6584c35..616140f426f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -53,16 +53,28 @@ struct PreparedAddNode { struct PreparedFirstAddDoc : public PrepareResult {}; -struct PreparedAddDoc : public PrepareResult { +struct PreparedAddDoc final : public PrepareResult { using ReadGuard = vespalib::GenerationHandler::Guard; uint32_t docid; ReadGuard read_guard; std::vector<PreparedAddNode> nodes; PreparedAddDoc(uint32_t docid_in, ReadGuard read_guard_in) noexcept; - ~PreparedAddDoc(); + ~PreparedAddDoc() override; PreparedAddDoc(PreparedAddDoc&& other) noexcept; }; } + +using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; +struct SelectResult { + HnswTraversalCandidateVector used; + LinkArray unused; + SelectResult() noexcept; + SelectResult(const SelectResult &) = delete; + SelectResult & operator=(const SelectResult &) = delete; + SelectResult(SelectResult &&) noexcept = default; + ~SelectResult(); +}; + template <HnswIndexType type> class HnswIndex : public NearestNeighborIndex { public: @@ -85,7 +97,6 @@ protected: using LinkArrayStore = typename GraphType::LinkArrayStore; using LinkArrayRef = typename GraphType::LinkArrayRef; - using LinkArray = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; using LevelArrayRef = typename GraphType::LevelArrayRef; @@ -125,11 +136,6 @@ protected: * Used by select_neighbors_heuristic(). */ bool have_closer_distance(HnswTraversalCandidate candidate, const HnswTraversalCandidateVector& curr_result) const; - struct SelectResult { - HnswTraversalCandidateVector used; - LinkArray unused; - ~SelectResult() {} - }; template <typename HnswCandidateVectorT> SelectResult select_neighbors_heuristic(const HnswCandidateVectorT& neighbors, uint32_t max_links) const; template <typename HnswCandidateVectorT> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.cpp new file mode 100644 index 00000000000..5f104c18cf7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.cpp @@ -0,0 +1,10 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hnsw_test_node.h" + +namespace search::tensor { + +HnswTestNode::HnswTestNode() noexcept = default; +HnswTestNode::~HnswTestNode() = default; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.h index dd0e61ec9e8..ff86a65967c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_test_node.h @@ -2,6 +2,7 @@ #pragma once +#include <cstdint> #include <vector> namespace search::tensor { @@ -19,14 +20,18 @@ private: LevelArray _levels; public: - HnswTestNode() : _levels() {} - HnswTestNode(const LinkArray& level_0) : _levels() { _levels.push_back(level_0); } - HnswTestNode(const LevelArray& levels_in) : _levels(levels_in) {} - bool empty() const { return _levels.empty(); } - size_t size() const { return _levels.size(); } - const LevelArray& levels() const { return _levels; } - const LinkArray& level(size_t idx) const { return _levels[idx]; } - bool operator==(const HnswTestNode& rhs) { + HnswTestNode() noexcept; + HnswTestNode(const HnswTestNode &) = delete; + HnswTestNode & operator=(const HnswTestNode &) = delete; + HnswTestNode(HnswTestNode &&) noexcept = default; + ~HnswTestNode(); + HnswTestNode(LinkArray&& level_0) : _levels() { _levels.push_back(std::move(level_0)); } + HnswTestNode(LevelArray&& levels_in) : _levels(std::move(levels_in)) {} + bool empty() const noexcept { return _levels.empty(); } + size_t size() const noexcept { return _levels.size(); } + const LevelArray& levels() const noexcept { return _levels; } + const LinkArray& level(size_t idx) const noexcept { return _levels[idx]; } + bool operator==(const HnswTestNode& rhs) noexcept { return _levels == rhs._levels; } }; diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index 1f2da032619..b48ec93c10e 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -21,7 +21,7 @@ class SerializedTensorRef; */ class ITensorAttribute : public DocVectorAccess { public: - virtual ~ITensorAttribute() {} + virtual ~ITensorAttribute() = default; virtual std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docId) const = 0; virtual std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const = 0; virtual vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp index 5ad6224f6d4..223a0a5750f 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp @@ -28,9 +28,7 @@ ImportedTensorAttributeVectorReadGuard::ImportedTensorAttributeVectorReadGuard(s { } -ImportedTensorAttributeVectorReadGuard::~ImportedTensorAttributeVectorReadGuard() -{ -} +ImportedTensorAttributeVectorReadGuard::~ImportedTensorAttributeVectorReadGuard() = default; const ITensorAttribute * ImportedTensorAttributeVectorReadGuard::asTensorAttribute() const @@ -63,13 +61,13 @@ ImportedTensorAttributeVectorReadGuard::get_tensor_ref(uint32_t docid) const } vespalib::eval::TypedCells -ImportedTensorAttributeVectorReadGuard::get_vector(uint32_t docid, uint32_t subspace) const +ImportedTensorAttributeVectorReadGuard::get_vector(uint32_t docid, uint32_t subspace) const noexcept { return _target_tensor_attribute.get_vector(getTargetLid(docid), subspace); } search::tensor::VectorBundle -ImportedTensorAttributeVectorReadGuard::get_vectors(uint32_t docid) const +ImportedTensorAttributeVectorReadGuard::get_vectors(uint32_t docid) const noexcept { return _target_tensor_attribute.get_vectors(getTargetLid(docid)); } diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index e07de5486b6..5e6bf8961df 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -27,7 +27,7 @@ public: ImportedTensorAttributeVectorReadGuard(std::shared_ptr<MetaStoreReadGuard> targetMetaStoreReadGuard, const attribute::ImportedAttributeVector &imported_attribute, bool stableEnumGuard); - ~ImportedTensorAttributeVectorReadGuard(); + ~ImportedTensorAttributeVectorReadGuard() override; const ITensorAttribute *asTensorAttribute() const override; @@ -45,8 +45,8 @@ public: bool supports_get_serialized_tensor_ref() const override; uint32_t get_num_docs() const override { return getNumDocs(); } - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; - VectorBundle get_vectors(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override; + VectorBundle get_vectors(uint32_t docid) const noexcept override; const vespalib::eval::ValueType &getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp index 3645c511b01..c42242d8dc8 100644 --- a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp +++ b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp @@ -4,7 +4,6 @@ #include "temporary_vector_store.h" #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <cmath> -#include <mutex> #include <variant> using vespalib::eval::Int8Float; @@ -12,7 +11,7 @@ using vespalib::eval::Int8Float; namespace search::tensor { template<typename FloatType, bool extra_dim> -class BoundMipsDistanceFunction : public BoundDistanceFunction { +class BoundMipsDistanceFunction final : public BoundDistanceFunction { mutable TemporaryVectorStore<FloatType> _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs_vector; const vespalib::hwaccelrated::IAccelrated & _computer; @@ -24,7 +23,7 @@ class BoundMipsDistanceFunction : public BoundDistanceFunction { static const float *cast(const float * p) { return p; } static const int8_t *cast(const Int8Float * p) { return reinterpret_cast<const int8_t *>(p); } public: - BoundMipsDistanceFunction(const vespalib::eval::TypedCells& lhs, MaximumSquaredNormStore& sq_norm_store) + BoundMipsDistanceFunction(TypedCells lhs, MaximumSquaredNormStore& sq_norm_store) : BoundDistanceFunction(), _tmpSpace(lhs.size), _lhs_vector(_tmpSpace.storeLhs(lhs)), @@ -44,7 +43,7 @@ public: return _lhs_extra_dim; } - double calc(const vespalib::eval::TypedCells &rhs) const override { + double calc(TypedCells rhs) const noexcept override { vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); const FloatType * a = _lhs_vector.data(); const FloatType * b = rhs_vector.data(); @@ -58,32 +57,32 @@ public: } return -dp; } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { return threshold; } - double to_rawscore(double distance) const override { + double to_rawscore(double distance) const noexcept override { return -distance; } - double to_distance(double rawscore) const override { + double to_distance(double rawscore) const noexcept override { return -rawscore; } - double min_rawscore() const override { + double min_rawscore() const noexcept override { return std::numeric_limits<double>::lowest(); } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + double calc_with_limit(TypedCells rhs, double) const noexcept override { return calc(rhs); } }; template<typename FloatType> BoundDistanceFunction::UP -MipsDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { +MipsDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) { return std::make_unique<BoundMipsDistanceFunction<FloatType, false>>(lhs, *_sq_norm_store); } template<typename FloatType> BoundDistanceFunction::UP -MipsDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +MipsDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) { return std::make_unique<BoundMipsDistanceFunction<FloatType, true>>(lhs, *_sq_norm_store); }; diff --git a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h index 63b2a83c1b5..67a6eb58de0 100644 --- a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h +++ b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.h @@ -45,7 +45,7 @@ public: : _sq_norm_store(std::make_shared<MaximumSquaredNormStore>()) { } - ~MipsDistanceFunctionFactoryBase() = default; + ~MipsDistanceFunctionFactoryBase() override = default; MaximumSquaredNormStore& get_max_squared_norm_store() noexcept { return *_sq_norm_store; } }; @@ -59,12 +59,11 @@ public: template<typename FloatType> class MipsDistanceFunctionFactory : public MipsDistanceFunctionFactoryBase { public: - MipsDistanceFunctionFactory() : MipsDistanceFunctionFactoryBase() { } - ~MipsDistanceFunctionFactory() = default; + MipsDistanceFunctionFactory() noexcept = default; + ~MipsDistanceFunctionFactory() override = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp index 931fd3edb06..4bc90001227 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.cpp @@ -2,6 +2,7 @@ #include "prenormalized_angular_distance.h" #include "temporary_vector_store.h" +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> using vespalib::typify_invoke; using vespalib::eval::TypifyCellType; @@ -9,14 +10,14 @@ using vespalib::eval::TypifyCellType; namespace search::tensor { template<typename FloatType> -class BoundPrenormalizedAngularDistance : public BoundDistanceFunction { +class BoundPrenormalizedAngularDistance final : public BoundDistanceFunction { private: const vespalib::hwaccelrated::IAccelrated & _computer; mutable TemporaryVectorStore<FloatType> _tmpSpace; const vespalib::ConstArrayRef<FloatType> _lhs; double _lhs_norm_sq; public: - BoundPrenormalizedAngularDistance(const vespalib::eval::TypedCells& lhs) + explicit BoundPrenormalizedAngularDistance(TypedCells lhs) : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()), _tmpSpace(lhs.size), _lhs(_tmpSpace.storeLhs(lhs)) @@ -27,23 +28,21 @@ public: _lhs_norm_sq = 1.0; } } - double calc(const vespalib::eval::TypedCells& rhs) const override { - size_t sz = _lhs.size(); + double calc(TypedCells rhs) const noexcept override { vespalib::ConstArrayRef<FloatType> rhs_vector = _tmpSpace.convertRhs(rhs); - assert(sz == rhs_vector.size()); auto a = _lhs.data(); auto b = rhs_vector.data(); - double dot_product = _computer.dotProduct(a, b, sz); + double dot_product = _computer.dotProduct(a, b, _lhs.size()); double distance = _lhs_norm_sq - dot_product; return distance; } - double convert_threshold(double threshold) const override { + double convert_threshold(double threshold) const noexcept override { double cosine_similarity = 1.0 - threshold; double dot_product = cosine_similarity * _lhs_norm_sq; double distance = _lhs_norm_sq - dot_product; return distance; } - double to_rawscore(double distance) const override { + double to_rawscore(double distance) const noexcept override { double dot_product = _lhs_norm_sq - distance; double cosine_similarity = dot_product / _lhs_norm_sq; // should be in in range [-1,1] but roundoff may cause problems: @@ -53,7 +52,7 @@ public: double score = 1.0 / (1.0 + cosine_distance); return score; } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { + double calc_with_limit(TypedCells rhs, double) const noexcept override { return calc(rhs); } }; @@ -63,14 +62,14 @@ template class BoundPrenormalizedAngularDistance<double>; template <typename FloatType> BoundDistanceFunction::UP -PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_query_vector(const vespalib::eval::TypedCells& lhs) { +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_query_vector(TypedCells lhs) { using DFT = BoundPrenormalizedAngularDistance<FloatType>; return std::make_unique<DFT>(lhs); } template <typename FloatType> BoundDistanceFunction::UP -PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_insertion_vector(const vespalib::eval::TypedCells& lhs) { +PrenormalizedAngularDistanceFunctionFactory<FloatType>::for_insertion_vector(TypedCells lhs) { using DFT = BoundPrenormalizedAngularDistance<FloatType>; return std::make_unique<DFT>(lhs); } diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h index 0f647547e08..7e3a8c2c676 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h @@ -2,11 +2,7 @@ #pragma once -#include "distance_function.h" -#include "bound_distance_function.h" #include "distance_function_factory.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace search::tensor { @@ -18,8 +14,8 @@ template <typename FloatType> class PrenormalizedAngularDistanceFunctionFactory : public DistanceFunctionFactory { public: PrenormalizedAngularDistanceFunctionFactory() = default; - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override; - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override; + BoundDistanceFunction::UP for_query_vector(TypedCells lhs) override; + BoundDistanceFunction::UP for_insertion_vector(TypedCells lhs) override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 75927112b89..3c1bb51f4ea 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -42,7 +42,7 @@ SerializedFastValueAttribute::supports_get_serialized_tensor_ref() const } vespalib::eval::TypedCells -SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) const +SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) const noexcept { EntryRef ref = acquire_entry_ref(docid); auto vectors = _tensorBufferStore.get_vectors(ref); @@ -50,7 +50,7 @@ SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) cons } VectorBundle -SerializedFastValueAttribute::get_vectors(uint32_t docid) const +SerializedFastValueAttribute::get_vectors(uint32_t docid) const noexcept { EntryRef ref = acquire_entry_ref(docid); return _tensorBufferStore.get_vectors(ref); diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h index 386b0d91add..43b5a23d176 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h @@ -27,8 +27,8 @@ public: bool supports_get_serialized_tensor_ref() const override; // Implements DocVectorAccess - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; - VectorBundle get_vectors(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override; + VectorBundle get_vectors(uint32_t docid) const noexcept override; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp index ff07f245de4..4753e9d7c87 100644 --- a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.cpp @@ -2,10 +2,6 @@ #include "temporary_vector_store.h" -#include <vespa/log/log.h> - -LOG_SETUP(".searchlib.tensor.temporary_vector_store"); - using vespalib::ConstArrayRef; using vespalib::ArrayRef; using vespalib::eval::CellType; @@ -17,7 +13,7 @@ namespace { template<typename FromType, typename ToType> ConstArrayRef<ToType> -convert_cells(ArrayRef<ToType> space, TypedCells cells) +convert_cells(ArrayRef<ToType> space, TypedCells cells) noexcept { assert(cells.size == space.size()); auto old_cells = cells.typify<FromType>(); @@ -32,7 +28,7 @@ convert_cells(ArrayRef<ToType> space, TypedCells cells) template <typename ToType> struct ConvertCellsSelector { - template <typename FromType> static auto invoke(ArrayRef<ToType> dst, TypedCells src) { + template <typename FromType> static auto invoke(ArrayRef<ToType> dst, TypedCells src) noexcept { return convert_cells<FromType, ToType>(dst, src); } }; @@ -41,8 +37,7 @@ struct ConvertCellsSelector template <typename FloatType> ConstArrayRef<FloatType> -TemporaryVectorStore<FloatType>::internal_convert(TypedCells cells, size_t offset) { - LOG_ASSERT(cells.size * 2 == _tmpSpace.size()); +TemporaryVectorStore<FloatType>::internal_convert(TypedCells cells, size_t offset) noexcept { ArrayRef<FloatType> where(_tmpSpace.data() + offset, cells.size); using MyTypify = vespalib::eval::TypifyCellType; using MySelector = ConvertCellsSelector<FloatType>; diff --git a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h index ad5bdf3ed3a..3dc237c85a4 100644 --- a/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h +++ b/searchlib/src/vespa/searchlib/tensor/temporary_vector_store.h @@ -2,10 +2,7 @@ #pragma once -#include <memory> -#include <vespa/eval/eval/cell_type.h> #include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/util/arrayref.h> namespace search::tensor { @@ -13,14 +10,15 @@ namespace search::tensor { template <typename FloatType> class TemporaryVectorStore { private: + using TypedCells = vespalib::eval::TypedCells; std::vector<FloatType> _tmpSpace; - vespalib::ConstArrayRef<FloatType> internal_convert(vespalib::eval::TypedCells cells, size_t offset); + vespalib::ConstArrayRef<FloatType> internal_convert(TypedCells cells, size_t offset) noexcept; public: - TemporaryVectorStore(size_t vectorSize) : _tmpSpace(vectorSize * 2) {} - vespalib::ConstArrayRef<FloatType> storeLhs(vespalib::eval::TypedCells cells) { + explicit TemporaryVectorStore(size_t vectorSize) noexcept : _tmpSpace(vectorSize * 2) {} + vespalib::ConstArrayRef<FloatType> storeLhs(TypedCells cells) noexcept { return internal_convert(cells, 0); } - vespalib::ConstArrayRef<FloatType> convertRhs(vespalib::eval::TypedCells cells) { + vespalib::ConstArrayRef<FloatType> convertRhs(TypedCells cells) { if (vespalib::eval::get_cell_type<FloatType>() == cells.type) [[likely]] { return cells.unsafe_typify<FloatType>(); } else { diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp index 5d37009b611..28c4099c38b 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_loader.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/objects/nbostream.h> #include <mutex> #include <condition_variable> diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_saver.cpp index 0fa5b2daab8..14ebe9329e4 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute_saver.cpp @@ -6,6 +6,7 @@ #include "tensor_attribute_constants.h" #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> +#include <vespa/vespalib/objects/nbostream.h> #include <cassert> using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 9a2192cf736..b93249b7e21 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -105,18 +105,18 @@ public: vespalib::eval::TypedCells get_empty_subspace() const noexcept { return _empty.cells(); } - VectorBundle get_vectors(vespalib::ConstArrayRef<char> buf) const { + VectorBundle get_vectors(vespalib::ConstArrayRef<char> buf) const noexcept { auto num_subspaces = get_num_subspaces(buf); auto cells_mem_size = get_cells_mem_size(num_subspaces); auto aligner = select_aligner(cells_mem_size); - return VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type); + return {buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type}; } - SerializedTensorRef get_serialized_tensor_ref(vespalib::ConstArrayRef<char> buf) const { + SerializedTensorRef get_serialized_tensor_ref(vespalib::ConstArrayRef<char> buf) const noexcept { auto num_subspaces = get_num_subspaces(buf); auto cells_mem_size = get_cells_mem_size(num_subspaces); auto aligner = select_aligner(cells_mem_size); vespalib::ConstArrayRef<vespalib::string_id> labels(reinterpret_cast<const vespalib::string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); - return SerializedTensorRef(VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type), _num_mapped_dimensions, labels); + return {VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type), _num_mapped_dimensions, labels}; } bool is_dense() const noexcept { return _num_mapped_dimensions == 0; } }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp index 600a7e92ed0..aaf5466daef 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/objects/nbostream.h> using document::DeserializeException; using vespalib::alloc::MemoryAllocator; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h index c8d96adc220..07275c77566 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h @@ -41,16 +41,16 @@ public: vespalib::eval::TypedCells get_empty_subspace() const noexcept { return _ops.get_empty_subspace(); } - VectorBundle get_vectors(EntryRef ref) const { + VectorBundle get_vectors(EntryRef ref) const noexcept { if (!ref.valid()) { - return VectorBundle(); + return {}; } auto buf = _array_store.get(ref); return _ops.get_vectors(buf); } - SerializedTensorRef get_serialized_tensor_ref(EntryRef ref) const { + SerializedTensorRef get_serialized_tensor_ref(EntryRef ref) const noexcept { if (!ref.valid()) { - return SerializedTensorRef(); + return {}; } auto buf = _array_store.get(ref); return _ops.get_serialized_tensor_ref(buf); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp index 1f85dba6afe..716d54d0a71 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp @@ -86,20 +86,20 @@ TensorExtAttribute::getExtendInterface() } TypedCells -TensorExtAttribute::get_vector(uint32_t docid, uint32_t subspace) const +TensorExtAttribute::get_vector(uint32_t docid, uint32_t subspace) const noexcept { auto vectors = get_vectors(docid); return (subspace < vectors.subspaces()) ? vectors.cells(subspace) : _empty.cells(); } VectorBundle -TensorExtAttribute::get_vectors(uint32_t docid) const +TensorExtAttribute::get_vectors(uint32_t docid) const noexcept { auto tensor = _data[docid]; if (tensor == nullptr) { - return VectorBundle(); + return {}; } - return VectorBundle(tensor->cells().data, tensor->index().size(), _subspace_type); + return {tensor->cells().data, static_cast<uint32_t>(tensor->index().size()), _subspace_type}; } std::unique_ptr<Value> diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h index 890b568c26e..0434c2ab65f 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h @@ -37,8 +37,8 @@ public: bool add(const vespalib::eval::Value& v, int32_t) override; IExtendAttribute* getExtendInterface() override; // DocVectorAccess API - vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; - VectorBundle get_vectors(uint32_t docid) const override; + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const noexcept override; + VectorBundle get_vectors(uint32_t docid) const noexcept override; // ITensorAttribute API std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docid) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/vector_bundle.h b/searchlib/src/vespa/searchlib/tensor/vector_bundle.h index 7ff7ea943de..e8b65c5b6b2 100644 --- a/searchlib/src/vespa/searchlib/tensor/vector_bundle.h +++ b/searchlib/src/vespa/searchlib/tensor/vector_bundle.h @@ -39,8 +39,7 @@ public: ~VectorBundle() = default; uint32_t subspaces() const noexcept { return _subspaces; } vespalib::eval::TypedCells cells(uint32_t subspace) const noexcept { - assert(subspace < _subspaces); - return vespalib::eval::TypedCells(static_cast<const char*>(_data) + _subspace_mem_size * subspace, _cell_type, _subspace_size); + return {static_cast<const char*>(_data) + _subspace_mem_size * subspace, _cell_type, _subspace_size}; } }; diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp index 58741efa9ec..e8e7e040b7e 100644 --- a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/direct_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> +#include <vespa/vespalib/objects/nbostream.h> using search::attribute::BasicType; using search::attribute::CollectionType; diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index ff74dc7a0e0..db8fe14d658 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -23,14 +23,17 @@ ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, bool isretryread = false; retry: - if (decodeContext.lastChunk()) + if (decodeContext.lastChunk()) { return; // Already reached end of file. - int remainingUnits = decodeContext.remainingUnits(); + } + int64_t remainingUnits = decodeContext.remainingUnits(); + assert(remainingUnits >= 0); // There's a good amount of data here already. if (remainingUnits > - static_cast<ssize_t>(ComprBuffer::minimumPadding())) //FIX! Tune + static_cast<ssize_t>(ComprBuffer::minimumPadding())) { //FIX! Tune return; + } // Assert that file read offset is aligned on unit boundary assert((static_cast<size_t>(fileReadByteOffset) & @@ -47,9 +50,9 @@ ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, // Continuation reads starts at aligned boundary. assert(remainingUnits == 0 || padBeforeUnits == 0); - if (readAll) + if (readAll) { stopOffset = fileSize << 3; - else if (!isretryread) { + } else if (!isretryread) { stopOffset += 8 * cbuf.getUnitBitSize(); // XXX: Magic integer // Realign stop offset to direct IO alignment boundary uint64_t fileDirectIOBitAlign = @@ -93,20 +96,19 @@ ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, fileReadByteOffset -= padBeforeUnits * cbuf.getUnitSize(); file.SetPosition(fileReadByteOffset); } - int readUnits0 = 0; - if (readBits > 0) - readUnits0 = static_cast<int>((readBits + cbuf.getUnitBitSize() - 1) / - cbuf.getUnitBitSize()); + size_t readUnits0 = 0; + if (readBits > 0) { + readUnits0 = (readBits + cbuf.getUnitBitSize() - 1) / cbuf.getUnitBitSize(); + } // Try to align end of read to an alignment boundary - int readUnits = cbuf.getAligner().adjustElements(fileReadByteOffset / - cbuf.getUnitSize(), readUnits0); - if (readUnits < readUnits0) + size_t readUnits = cbuf.getAligner().adjustElements(fileReadByteOffset / cbuf.getUnitSize(), readUnits0); + if (readUnits < readUnits0) { isMore = true; + } if (readUnits > 0) { - int64_t padBytes = fileReadByteOffset + - static_cast<int64_t>(readUnits) * cbuf.getUnitSize() - fileSize; + int64_t padBytes = fileReadByteOffset + readUnits * cbuf.getUnitSize() - fileSize; if (!isMore && padBytes > 0) { // Pad reading of file written with smaller unit size with // NUL bytes. @@ -115,17 +117,18 @@ ComprFileReadBase::ReadComprBuffer(uint64_t stopOffset, readUnits * cbuf.getUnitSize() - padBytes, 0, padBytes); - } else + } else { file.ReadBuf(cbuf.getComprBuf(), readUnits * cbuf.getUnitSize()); + } } // If at end of file then add units of zero bits as padding - if (!isMore) + if (!isMore) { memset(reinterpret_cast<char *>(cbuf.getComprBuf()) + readUnits * cbuf.getUnitSize(), 0, cbuf.getUnitSize() * ComprBuffer::minimumPadding()); + } - assert(remainingUnits + readUnits >= 0); decodeContext.afterRead(reinterpret_cast<char *>(cbuf.getComprBuf()) + (padBeforeUnits - remainingUnits) * static_cast<int32_t>(cbuf.getUnitSize()), @@ -343,7 +346,7 @@ ComprFileReadContext::setPosition(uint64_t newPosition) } void -ComprFileReadContext::allocComprBuf(unsigned int comprBufSize, size_t preferredFileAlignment) +ComprFileReadContext::allocComprBuf(size_t comprBufSize, size_t preferredFileAlignment) { ComprBuffer::allocComprBuf(comprBufSize, preferredFileAlignment, _file, true); } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 8f8cffaffd6..6a0b72ce7e5 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -32,7 +32,7 @@ public: * Get remaining units in buffer (e.g. _realValE - _valI) */ - virtual int32_t remainingUnits() const = 0; + virtual int64_t remainingUnits() const = 0; /** * Get unit ptr (e.g. _valI) from decode context. @@ -51,7 +51,7 @@ public: virtual uint64_t getBitPos(int bitOffset, uint64_t bufferEndFilePos) const = 0; virtual uint64_t getBitPosV() const = 0; virtual void skipBits(int bits) = 0; - virtual void adjUnitPtr(int newRemainingUnits) = 0; + virtual void adjUnitPtr(int64_t newRemainingUnits) = 0; virtual void emptyBuffer(uint64_t newBitPosition) = 0; /** @@ -105,7 +105,7 @@ public: void readComprBuffer(uint64_t stopOffset, bool readAll); void readComprBuffer(); void setPosition(uint64_t newPosition); - void allocComprBuf(unsigned int comprBufSize, size_t preferredFileAlignment); + void allocComprBuf(size_t comprBufSize, size_t preferredFileAlignment); void setDecodeContext(ComprFileDecodeContext *decodeContext) { _decodeContext = decodeContext; } ComprFileDecodeContext *getDecodeContext() const { return _decodeContext; } void setFile(FastOS_FileInterface *file) { _file = file; } diff --git a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp index c20f9570ef8..dd64f8637e7 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp @@ -36,6 +36,7 @@ #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/objects/nbostream.h> using document::ArrayFieldValue; using document::BoolFieldValue; diff --git a/security-utils/src/main/java/com/yahoo/security/AutoReloadingX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/AutoReloadingX509KeyManager.java index be1940441d4..c98e0791f2d 100644 --- a/security-utils/src/main/java/com/yahoo/security/AutoReloadingX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/AutoReloadingX509KeyManager.java @@ -6,7 +6,6 @@ import javax.net.ssl.X509ExtendedKeyManager; import java.io.IOException; import java.io.UncheckedIOException; import java.net.Socket; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; @@ -14,7 +13,7 @@ import java.security.Principal; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Duration; -import java.util.Arrays; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -60,7 +59,7 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen X509ExtendedKeyManager manager = mutableX509KeyManager.currentManager(); X509Certificate[] certificateChain = manager.getCertificateChain(CERTIFICATE_ALIAS); PrivateKey privateKey = manager.getPrivateKey(CERTIFICATE_ALIAS); - return new X509CertificateWithKey(Arrays.asList(certificateChain), privateKey); + return new X509CertificateWithKey(List.of(certificateChain), privateKey); } private static KeyStore createKeystore(Path privateKey, Path certificateChain) { @@ -68,8 +67,8 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen return KeyStoreBuilder.withType(KeyStoreType.PKCS12) .withKeyEntry( CERTIFICATE_ALIAS, - KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKey), StandardCharsets.UTF_8)), - X509CertificateUtils.certificateListFromPem(new String(Files.readAllBytes(certificateChain), StandardCharsets.UTF_8))) + KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKey)), + X509CertificateUtils.certificateListFromPem(Files.readString(certificateChain))) .build(); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/security-utils/src/main/java/com/yahoo/security/KeyManagerUtils.java b/security-utils/src/main/java/com/yahoo/security/KeyManagerUtils.java index ab584dbb48e..f25118029da 100644 --- a/security-utils/src/main/java/com/yahoo/security/KeyManagerUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/KeyManagerUtils.java @@ -27,7 +27,7 @@ public class KeyManagerUtils { .filter(manager -> manager instanceof X509ExtendedKeyManager) .map(X509ExtendedKeyManager.class::cast) .findFirst() - .orElseThrow(() -> new RuntimeException("No X509ExtendedKeyManager in " + Arrays.asList(keyManagers))); + .orElseThrow(() -> new RuntimeException("No X509ExtendedKeyManager in " + List.of(keyManagers))); } catch (GeneralSecurityException e) { throw new RuntimeException(e); } diff --git a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java index c4c01ca130c..0901ea5931f 100644 --- a/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/KeyStoreBuilder.java @@ -15,8 +15,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; -import static java.util.Collections.singletonList; - /** * @author bjorncs */ @@ -53,7 +51,7 @@ public class KeyStoreBuilder { } public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, char[] password, X509Certificate certificate) { - return withKeyEntry(alias, privateKey, password, singletonList(certificate)); + return withKeyEntry(alias, privateKey, password, List.of(certificate)); } public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, X509Certificate certificate) { diff --git a/security-utils/src/main/java/com/yahoo/security/Pkcs10Csr.java b/security-utils/src/main/java/com/yahoo/security/Pkcs10Csr.java index 78a00246d38..d1c9ae582b7 100644 --- a/security-utils/src/main/java/com/yahoo/security/Pkcs10Csr.java +++ b/security-utils/src/main/java/com/yahoo/security/Pkcs10Csr.java @@ -15,8 +15,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import static java.util.Collections.emptyList; - /** * @author bjorncs */ @@ -40,7 +38,7 @@ public class Pkcs10Csr { return getExtensions() .map(extensions -> GeneralNames.fromExtensions(extensions, Extension.subjectAlternativeName)) .map(SubjectAlternativeName::fromGeneralNames) - .orElse(emptyList()); + .orElse(List.of()); } /** @@ -57,7 +55,7 @@ public class Pkcs10Csr { .map(extensions -> Arrays.stream(extensions.getExtensionOIDs()) .map(ASN1ObjectIdentifier::getId) .toList()) - .orElse(emptyList()); + .orElse(List.of()); } diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java index cedad3afc9b..8fecbb72a43 100644 --- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java @@ -18,8 +18,6 @@ import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.util.List; -import static java.util.Collections.singletonList; - /** * A builder for {@link SSLContext}. * @@ -48,7 +46,7 @@ public class SslContextBuilder { } public SslContextBuilder withTrustStore(X509Certificate caCertificate) { - return withTrustStore(singletonList(caCertificate)); + return withTrustStore(List.of(caCertificate)); } public SslContextBuilder withTrustStore(List<X509Certificate> caCertificates) { @@ -66,7 +64,7 @@ public class SslContextBuilder { } public SslContextBuilder withKeyStore(PrivateKey privateKey, X509Certificate certificate) { - return withKeyStore(privateKey, singletonList(certificate)); + return withKeyStore(privateKey, List.of(certificate)); } public SslContextBuilder withKeyStore(PrivateKey privateKey, List<X509Certificate> certificates) { diff --git a/security-utils/src/main/java/com/yahoo/security/TrustManagerUtils.java b/security-utils/src/main/java/com/yahoo/security/TrustManagerUtils.java index 0b06584afb7..2d3adffea87 100644 --- a/security-utils/src/main/java/com/yahoo/security/TrustManagerUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/TrustManagerUtils.java @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; - import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; @@ -29,7 +26,7 @@ public class TrustManagerUtils { .filter(manager -> manager instanceof X509ExtendedTrustManager) .map(X509ExtendedTrustManager.class::cast) .findFirst() - .orElseThrow(() -> new RuntimeException("No X509ExtendedTrustManager in " + Arrays.asList(trustManagers))); + .orElseThrow(() -> new RuntimeException("No X509ExtendedTrustManager in " + List.of(trustManagers))); } catch (GeneralSecurityException e) { throw new RuntimeException(e); } diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 171a8e890d0..f615ff2e832 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -34,7 +34,6 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Random; @@ -150,7 +149,7 @@ public class X509CertificateUtils { public static List<SubjectAlternativeName> getSubjectAlternativeNames(X509Certificate certificate) { try { byte[] extensionValue = certificate.getExtensionValue(SUBJECT_ALTERNATIVE_NAMES.getOId()); - if (extensionValue == null) return Collections.emptyList(); + if (extensionValue == null) return List.of(); ASN1Encodable asn1Encodable = ASN1Primitive.fromByteArray(extensionValue); if (asn1Encodable instanceof ASN1OctetString) { asn1Encodable = ASN1Primitive.fromByteArray(((ASN1OctetString) asn1Encodable).getOctets()); diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java index e80d3840bce..afd5fa315a6 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java @@ -3,7 +3,6 @@ package com.yahoo.security; import java.security.PrivateKey; import java.security.cert.X509Certificate; -import java.util.Collections; import java.util.List; /** @@ -18,7 +17,7 @@ public class X509CertificateWithKey { private final PrivateKey privateKey; public X509CertificateWithKey(X509Certificate certificate, PrivateKey privateKey) { - this(Collections.singletonList(certificate), privateKey); + this(List.of(certificate), privateKey); } public X509CertificateWithKey(List<X509Certificate> certificate, PrivateKey privateKey) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index 4e810c2d304..c2092d0c22b 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -64,7 +64,7 @@ public class DefaultTlsContext implements TlsContext { String.format("None of the accepted ciphers are supported (supported=%s, accepted=%s)", supportedCiphers, acceptedCiphers)); } - log.log(Level.FINE, () -> String.format("Allowed cipher suites that are supported: %s", Arrays.asList(allowedCiphers))); + log.log(Level.FINE, () -> String.format("Allowed cipher suites that are supported: %s", List.of(allowedCiphers))); return allowedCiphers; } diff --git a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrTest.java b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrTest.java index 04282ceaac7..b78fa16ae56 100644 --- a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrTest.java +++ b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrTest.java @@ -5,7 +5,6 @@ import org.junit.jupiter.api.Test; import javax.security.auth.x500.X500Principal; import java.security.KeyPair; -import java.util.Arrays; import java.util.List; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; @@ -27,7 +26,7 @@ public class Pkcs10CsrTest { .addSubjectAlternativeName(san1) .addSubjectAlternativeName(san2) .build(); - assertEquals(Arrays.asList(san1, san2), csr.getSubjectAlternativeNames()); + assertEquals(List.of(san1, san2), csr.getSubjectAlternativeNames()); } @Test @@ -49,7 +48,7 @@ public class Pkcs10CsrTest { .addSubjectAlternativeName("san") .setBasicConstraints(true, true) .build(); - List<String> expected = Arrays.asList(Extension.BASIC_CONSTRAINTS.getOId(), Extension.SUBJECT_ALTERNATIVE_NAMES.getOId()); + List<String> expected = List.of(Extension.BASIC_CONSTRAINTS.getOId(), Extension.SUBJECT_ALTERNATIVE_NAMES.getOId()); List<String> actual = csr.getExtensionOIds(); assertEquals(expected, actual); } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateBuilderTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateBuilderTest.java index d91c9fc23f2..5d5cc7b7fa5 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateBuilderTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateBuilderTest.java @@ -10,8 +10,8 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class X509CertificateBuilderTest { public static Collection<Object[]> data() { - return Arrays.asList(new Object[][]{ + return List.of(new Object[][]{ {KeyAlgorithm.RSA, 2048, SignatureAlgorithm.SHA512_WITH_RSA}, {KeyAlgorithm.EC, 256, SignatureAlgorithm.SHA512_WITH_ECDSA}}); } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 6ac71a264b3..e67dce9fd8f 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -9,7 +9,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.List; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; @@ -42,7 +41,7 @@ public class X509CertificateUtilsTest { X509Certificate cert1 = TestUtils.createCertificate(keypair, subject1); X500Principal subject2 = new X500Principal("CN=myservice2"); X509Certificate cert2 = TestUtils.createCertificate(keypair, subject2); - List<X509Certificate> certificateList = Arrays.asList(cert1, cert2); + List<X509Certificate> certificateList = List.of(cert1, cert2); String pem = X509CertificateUtils.toPem(certificateList); List<X509Certificate> deserializedCertificateList = X509CertificateUtils.certificateListFromPem(pem); assertEquals(2, certificateList.size()); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/AuthorizedPeersTest.java b/security-utils/src/test/java/com/yahoo/security/tls/AuthorizedPeersTest.java index e6f3450332d..ee54a80f732 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/AuthorizedPeersTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/AuthorizedPeersTest.java @@ -3,11 +3,10 @@ package com.yahoo.security.tls; import org.junit.jupiter.api.Test; -import java.util.HashSet; +import java.util.List; +import java.util.Set; import static com.yahoo.security.tls.RequiredPeerCredential.Field.CN; -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -18,9 +17,9 @@ public class AuthorizedPeersTest { @Test void throws_exception_on_peer_policies_with_duplicate_names() { assertThrows(IllegalArgumentException.class, () -> { - PeerPolicy peerPolicy1 = new PeerPolicy("duplicate-name", singletonList(RequiredPeerCredential.of(CN, "mycfgserver"))); - PeerPolicy peerPolicy2 = new PeerPolicy("duplicate-name", singletonList(RequiredPeerCredential.of(CN, "myclient"))); - new AuthorizedPeers(new HashSet<>(asList(peerPolicy1, peerPolicy2))); + PeerPolicy peerPolicy1 = new PeerPolicy("duplicate-name", List.of(RequiredPeerCredential.of(CN, "mycfgserver"))); + PeerPolicy peerPolicy2 = new PeerPolicy("duplicate-name", List.of(RequiredPeerCredential.of(CN, "myclient"))); + new AuthorizedPeers(Set.of(peerPolicy1, peerPolicy2)); }); } diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java index ec7d5b8ca05..267e770050d 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java @@ -18,7 +18,6 @@ import static com.yahoo.security.X509CertificateBuilder.generateRandomSerialNumb import static java.time.Instant.EPOCH; import static java.time.temporal.ChronoUnit.DAYS; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; /** @@ -38,7 +37,7 @@ public class DefaultTlsContextTest { singleton( new PeerPolicy( "dummy-policy", - singletonList(RequiredPeerCredential.of(RequiredPeerCredential.Field.CN, "dummy"))))); + List.of(RequiredPeerCredential.of(RequiredPeerCredential.Field.CN, "dummy"))))); DefaultTlsContext tlsContext = DefaultTlsContext.of( diff --git a/security-utils/src/test/java/com/yahoo/security/tls/PeerAuthorizerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/PeerAuthorizerTest.java index 112cfa75102..91ce19574fe 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/PeerAuthorizerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/PeerAuthorizerTest.java @@ -23,9 +23,6 @@ import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static com.yahoo.security.tls.RequiredPeerCredential.Field.CN; import static com.yahoo.security.tls.RequiredPeerCredential.Field.SAN_DNS; import static com.yahoo.security.tls.RequiredPeerCredential.Field.SAN_URI; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toSet; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -45,13 +42,13 @@ public class PeerAuthorizerTest { RequiredPeerCredential sanRequirement = createRequiredCredential(SAN_DNS, "*.matching.san"); PeerAuthorizer authorizer = createPeerAuthorizer(createPolicy(POLICY_1, cnRequirement, sanRequirement)); - ConnectionAuthContext result = authorizer.authorizePeer(createCertificate("foo.matching.cn", asList("foo.matching.san", "foo.invalid.san"), emptyList())); + ConnectionAuthContext result = authorizer.authorizePeer(createCertificate("foo.matching.cn", List.of("foo.matching.san", "foo.invalid.san"), List.of())); assertAuthorized(result); assertThat(result.matchedPolicies()).containsOnly(POLICY_1); - assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", singletonList("foo.matching.san"), emptyList()))); - assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", asList("foo.matching.san", "foo.invalid.san"), emptyList()))); - assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.matching.cn", singletonList("foo.invalid.san"), emptyList()))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", List.of("foo.matching.san"), List.of()))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", List.of("foo.matching.san", "foo.invalid.san"), List.of()))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.matching.cn", List.of("foo.invalid.san"), List.of()))); } @Test @@ -64,7 +61,7 @@ public class PeerAuthorizerTest { createPolicy(POLICY_2, cnRequirement, sanRequirement)); ConnectionAuthContext result = peerAuthorizer - .authorizePeer(createCertificate("foo.matching.cn", singletonList("foo.matching.san"), emptyList())); + .authorizePeer(createCertificate("foo.matching.cn", List.of("foo.matching.san"), List.of())); assertAuthorized(result); assertThat(result.matchedPolicies()).containsOnly(POLICY_1, POLICY_2); } @@ -75,7 +72,7 @@ public class PeerAuthorizerTest { createPolicy(POLICY_1, createRequiredCredential(CN, "*.matching.cn")), createPolicy(POLICY_2, createRequiredCredential(SAN_DNS, "*.matching.san"))); - ConnectionAuthContext result = peerAuthorizer.authorizePeer(createCertificate("foo.invalid.cn", singletonList("foo.matching.san"), emptyList())); + ConnectionAuthContext result = peerAuthorizer.authorizePeer(createCertificate("foo.invalid.cn", List.of("foo.matching.san"), List.of())); assertAuthorized(result); assertThat(result.matchedPolicies()).containsOnly(POLICY_2); } @@ -89,9 +86,9 @@ public class PeerAuthorizerTest { PeerAuthorizer peerAuthorizer = createPeerAuthorizer( createPolicy(POLICY_1, cnSuffixRequirement, cnPrefixRequirement, sanPrefixRequirement, sanSuffixRequirement)); - assertAuthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", singletonList("matching.prefix.matching.suffix.san"), emptyList()))); - assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", singletonList("matching.prefix.invalid.suffix.san"), emptyList()))); - assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("invalid.prefix.matching.suffix.cn", singletonList("matching.prefix.matching.suffix.san"), emptyList()))); + assertAuthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", List.of("matching.prefix.matching.suffix.san"), List.of()))); + assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", List.of("matching.prefix.invalid.suffix.san"), List.of()))); + assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("invalid.prefix.matching.suffix.cn", List.of("matching.prefix.matching.suffix.san"), List.of()))); } @Test @@ -100,11 +97,11 @@ public class PeerAuthorizerTest { RequiredPeerCredential sanUriRequirement = createRequiredCredential(SAN_URI, "myscheme://my/*/uri"); PeerAuthorizer authorizer = createPeerAuthorizer(createPolicy(POLICY_1, cnRequirement, sanUriRequirement)); - ConnectionAuthContext result = authorizer.authorizePeer(createCertificate("foo.matching.cn", singletonList("foo.irrelevant.san"), singletonList("myscheme://my/matching/uri"))); + ConnectionAuthContext result = authorizer.authorizePeer(createCertificate("foo.matching.cn", List.of("foo.irrelevant.san"), List.of("myscheme://my/matching/uri"))); assertAuthorized(result); assertThat(result.matchedPolicies()).containsOnly(POLICY_1); - assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.matching.cn", emptyList(), singletonList("myscheme://my/nonmatching/url")))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.matching.cn", List.of(), List.of("myscheme://my/nonmatching/url")))); } @Test @@ -145,7 +142,7 @@ public class PeerAuthorizerTest { } private static PeerPolicy createPolicy(String name, RequiredPeerCredential... requiredCredentials) { - return new PeerPolicy(name, asList(requiredCredentials)); + return new PeerPolicy(name, List.of(requiredCredentials)); } private static PeerPolicy createPolicy(String name, List<Capability> caps, List<RequiredPeerCredential> creds) { diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializerTest.java index 1871bb43569..e7254c12985 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializerTest.java @@ -14,9 +14,8 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import static com.yahoo.security.tls.RequiredPeerCredential.Field.CN; @@ -43,14 +42,14 @@ public class TransportSecurityOptionsJsonSerializerTest { .withHostnameValidationDisabled(false) .withAuthorizedPeers( new AuthorizedPeers( - new LinkedHashSet<>(Arrays.asList( - new PeerPolicy("cfgserver", "cfgserver policy description", Arrays.asList( + new LinkedHashSet<>(List.of( + new PeerPolicy("cfgserver", "cfgserver policy description", List.of( RequiredPeerCredential.of(CN, "mycfgserver"), RequiredPeerCredential.of(SAN_DNS, "*.suffix.com"), RequiredPeerCredential.of(SAN_URI, "myscheme://resource/path/"))), new PeerPolicy("node", Optional.empty(), CapabilitySet.of(Capability.SLOBROK__API), - Collections.singletonList(RequiredPeerCredential.of(CN, "hostname"))))))) + List.of(RequiredPeerCredential.of(CN, "hostname"))))))) .build(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -67,8 +66,8 @@ public class TransportSecurityOptionsJsonSerializerTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) - .withAcceptedCiphers(Arrays.asList("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384")) - .withAcceptedProtocols(Collections.singletonList("TLSv1.2")) + .withAcceptedCiphers(List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384")) + .withAcceptedProtocols(List.of("TLSv1.2")) .withHostnameValidationDisabled(true) .build(); File outputFile = File.createTempFile("junit", null, tempDirectory); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java index 08e573fed7e..188a6a1568a 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java @@ -4,12 +4,10 @@ package com.yahoo.security.tls; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -22,8 +20,8 @@ public class TransportSecurityOptionsTest { private static final TransportSecurityOptions OPTIONS = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) - .withAcceptedCiphers(Arrays.asList("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384")) - .withAcceptedProtocols(Collections.singletonList("TLSv1.2")) + .withAcceptedCiphers(List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384")) + .withAcceptedProtocols(List.of("TLSv1.2")) .withHostnameValidationDisabled(true) .build(); @@ -35,7 +33,7 @@ public class TransportSecurityOptionsTest { @Test void can_read_options_from_json() throws IOException { - String tlsJson = new String(Files.readAllBytes(TEST_CONFIG_FILE), StandardCharsets.UTF_8); + String tlsJson = Files.readString(TEST_CONFIG_FILE); TransportSecurityOptions actualOptions = TransportSecurityOptions.fromJson(tlsJson); assertEquals(OPTIONS, actualOptions); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 1c2c57e9a77..bd05b1ab509 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.model.ModelGenerator; import com.yahoo.vespa.service.monitor.InfraApplicationApi; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -107,12 +106,12 @@ public abstract class InfraApplication implements InfraApplicationApi { // service name == service type for the first service of each type on each host serviceType.s(), serviceType.s(), - Collections.singletonList(portInfo), + List.of(portInfo), properties, configIdFor(hostname).s(), hostname.value()); - return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo)); + return new HostInfo(hostname.value(), List.of(serviceInfo)); } public ConfigId configIdFor(DomainName hostname) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java index 5bafc7ac4bd..7d89b86c07b 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -74,7 +73,7 @@ public class ServiceModel { ApplicationInstance previous = applicationInstances.put(instance.hostName(), application); if (previous != null && !previous.equals(application)) { throw new IllegalStateException("Major assumption broken: Multiple application instances contain host " + - instance.hostName().s() + ": " + Arrays.asList(previous, application)); + instance.hostName().s() + ": " + List.of(previous, application)); } serviceInstances diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index a5b8054536b..00c15add762 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -66,7 +66,7 @@ public class DuperModelTest { duperModel.add(application1); assertEquals(Optional.of(application1), duperModel.getApplicationInfo(id1)); - assertEquals(Arrays.asList(application1), duperModel.getApplicationInfos()); + assertEquals(List.of(application1), duperModel.getApplicationInfos()); assertEquals(1, duperModel.numberOfApplications()); duperModel.registerListener(listener1); @@ -86,7 +86,7 @@ public class DuperModelTest { assertEquals(Optional.empty(), duperModel.getApplicationInfo(id1)); verify(listener1, times(1)).applicationRemoved(id1); verifyNoMoreInteractions(listener1); - assertEquals(Arrays.asList(application2), duperModel.getApplicationInfos()); + assertEquals(List.of(application2), duperModel.getApplicationInfos()); duperModel.remove(id1); verifyNoMoreInteractions(listener1); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java index bef87a627f5..9c0c94cef27 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.service.slobrok.SlobrokMonitor; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -151,7 +150,7 @@ public class ExampleModel { * A bit unrealistic, but the port is the same on all hosts. */ ClusterBuilder addPort(int port, String... tags) { - portInfos.add(new PortInfo(port, Arrays.asList(tags))); + portInfos.add(new PortInfo(port, List.of(tags))); return this; } diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 844b912543c..92faa1ae670 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -113,6 +113,7 @@ model-evaluation-jar-with-dependencies.jar, model-integration-jar-with-dependencies.jar, container-onnxruntime.jar, + container-llama.jar, <!-- END config-model dependencies --> </discPreInstallBundle> </configuration> diff --git a/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java index 8da19fcb64b..5789a69a4db 100644 --- a/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java +++ b/standalone-container/src/main/java/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.java @@ -32,7 +32,6 @@ import java.net.URLClassLoader; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Dictionary; import java.util.Enumeration; import java.util.Hashtable; @@ -226,7 +225,7 @@ public final class ClassLoaderOsgiFramework implements OsgiFramework { @Override public Map<X509Certificate, List<X509Certificate>> getSignerCertificates(int signersType) { - return Collections.emptyMap(); + return Map.of(); } @Override diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java index 72f32ac04b8..9cd48ac85a8 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/CloudConfigInstallVariablesTest.java @@ -6,7 +6,6 @@ import org.junit.Test; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigModelsPluginDir; import static com.yahoo.container.standalone.CloudConfigInstallVariables.toConfigServer; @@ -39,7 +38,7 @@ public class CloudConfigInstallVariablesTest { assertEquals(2, parsed.length); List<String> hostNames = Arrays.stream(parsed).map(cs -> cs.hostName).toList(); - assertTrue(hostNames.containsAll(Arrays.asList("test1", "test2"))); + assertTrue(hostNames.containsAll(List.of("test1", "test2"))); } @Test(expected = IllegalArgumentException.class) diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java index 8d207388b42..00430e04858 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainer.java @@ -11,7 +11,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; import java.util.List; /** @@ -51,7 +50,7 @@ public class StandaloneContainer { private static void writeServicesXml(Path applicationPath, String servicesXml) throws IOException { Path path = applicationPath.resolve("services.xml"); - List<String> output = Arrays.asList("<?xml version=\"1.0\" encoding=\"utf-8\"?>", servicesXml); + List<String> output = List.of("<?xml version=\"1.0\" encoding=\"utf-8\"?>", servicesXml); Files.write(path, output, StandardCharsets.UTF_8); } } diff --git a/storage/src/tests/persistence/active_operations_stats_test.cpp b/storage/src/tests/persistence/active_operations_stats_test.cpp index bf91a84235a..71be34e3f54 100644 --- a/storage/src/tests/persistence/active_operations_stats_test.cpp +++ b/storage/src/tests/persistence/active_operations_stats_test.cpp @@ -28,6 +28,8 @@ public: std::shared_ptr<api::StorageMessage> createPut(uint64_t bucket, uint64_t docIdx); std::shared_ptr<api::StorageMessage> createGet(uint64_t bucket) const; + void SetUp() override; + void TearDown() override; void assert_active_operations_stats(const ActiveOperationsStats &stats, uint32_t exp_active_size, uint32_t exp_size_samples, uint32_t exp_latency_samples); void update_metrics(); void test_active_operations_stats(); @@ -41,16 +43,30 @@ ActiveOperationsStatsTest::ActiveOperationsStatsTest() metrics(), stripeId(0) { + // Initialization of members must happen in SetUp() since this test transitively + // depends on components modified by the superclass' SetUp() method. +} + +void +ActiveOperationsStatsTest::SetUp() +{ + FileStorTestFixture::SetUp(); setupPersistenceThreads(1); _node->setPersistenceProvider(std::make_unique<spi::dummy::DummyPersistence>(_node->getTypeRepo())); top.push_back(std::move(dummyManager)); top.open(); metrics.initDiskMetrics(1, 1); - filestorHandler = std::make_unique<FileStorHandlerImpl>(messageSender, metrics, - _node->getComponentRegister()); + filestorHandler = std::make_unique<FileStorHandlerImpl>(messageSender, metrics, _node->getComponentRegister()); filestorHandler->setGetNextMessageTimeout(20ms); } +void +ActiveOperationsStatsTest::TearDown() +{ + filestorHandler.reset(); + FileStorTestFixture::TearDown(); +} + ActiveOperationsStatsTest::~ActiveOperationsStatsTest() = default; std::shared_ptr<api::StorageMessage> diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index f1a8a286bbd..aa7c9fe995c 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST SOURCES deactivatebucketstest.cpp deletebuckettest.cpp + feed_operation_batching_test.cpp filestormanagertest.cpp filestormodifiedbucketstest.cpp mergeblockingtest.cpp @@ -18,6 +19,7 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST storage_testhostreporter storage_testpersistence_common GTest::GTest + absl::failure_signal_handler ) vespa_add_test( NAME storage_filestorage_gtest_runner_app COMMAND storage_filestorage_gtest_runner_app COST 50) diff --git a/storage/src/tests/persistence/filestorage/feed_operation_batching_test.cpp b/storage/src/tests/persistence/filestorage/feed_operation_batching_test.cpp new file mode 100644 index 00000000000..cf16123933b --- /dev/null +++ b/storage/src/tests/persistence/filestorage/feed_operation_batching_test.cpp @@ -0,0 +1,318 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <tests/persistence/common/filestortestfixture.h> +#include <tests/persistence/filestorage/forwardingmessagesender.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/persistence/filestorage/filestormetrics.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <gtest/gtest.h> + +using document::test::makeDocumentBucket; +using document::BucketId; +using document::DocumentId; +using namespace ::testing; + +namespace storage { + +struct FeedOperationBatchingTest : FileStorTestFixture { + DummyStorageLink _top; + std::unique_ptr<ForwardingMessageSender> _message_sender; + FileStorMetrics _metrics; + std::unique_ptr<FileStorHandler> _handler; + api::Timestamp _next_timestamp; + + FeedOperationBatchingTest(); + ~FeedOperationBatchingTest() override; + + void SetUp() override { + FileStorTestFixture::SetUp(); + // This silly little indirection is a work-around for the top-level link needing something + // below it to send _up_ into it, rather than directly receiving the messages itself. + auto message_receiver = std::make_unique<DummyStorageLink>(); + _message_sender = std::make_unique<ForwardingMessageSender>(*message_receiver); + _top.push_back(std::move(message_receiver)); + _top.open(); + _metrics.initDiskMetrics(1, 1); + // By default, sets up 1 thread with 1 stripe + _handler = std::make_unique<FileStorHandlerImpl>(*_message_sender, _metrics, _node->getComponentRegister()); + _handler->set_max_feed_op_batch_size(3); + } + + void TearDown() override { + _handler.reset(); + FileStorTestFixture::TearDown(); + } + + [[nodiscard]] static vespalib::string id_str_of(uint32_t bucket_idx, uint32_t doc_idx) { + return vespalib::make_string("id:foo:testdoctype1:n=%u:%u", bucket_idx, doc_idx); + } + + [[nodiscard]] static DocumentId id_of(uint32_t bucket_idx, uint32_t doc_idx) { + return DocumentId(id_str_of(bucket_idx, doc_idx)); + } + + void schedule_msg(const std::shared_ptr<api::StorageMessage>& msg) { + msg->setAddress(makeSelfAddress()); + _handler->schedule(msg); // takes shared_ptr by const ref, no point in moving + } + + void send_put(uint32_t bucket_idx, uint32_t doc_idx, uint32_t timestamp, vespalib::duration timeout) { + auto id = id_str_of(bucket_idx, doc_idx); + auto doc = _node->getTestDocMan().createDocument("foobar", id); + auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket({16, bucket_idx}), std::move(doc), timestamp); + cmd->setTimeout(timeout); + schedule_msg(cmd); + } + + void send_put(uint32_t bucket_idx, uint32_t doc_idx) { + send_put(bucket_idx, doc_idx, next_timestamp(), 60s); + } + + void send_puts(std::initializer_list<std::pair<uint32_t, uint32_t>> bucket_docs) { + for (const auto& bd : bucket_docs) { + send_put(bd.first, bd.second); + } + } + + void send_get(uint32_t bucket_idx, uint32_t doc_idx) { + auto id = id_of(bucket_idx, doc_idx); + auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket({16, bucket_idx}), id, document::AllFields::NAME); + schedule_msg(cmd); + } + + void send_remove(uint32_t bucket_idx, uint32_t doc_idx, uint32_t timestamp) { + auto id = id_of(bucket_idx, doc_idx); + auto cmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket({16, bucket_idx}), id, timestamp); + schedule_msg(cmd); + } + + void send_remove(uint32_t bucket_idx, uint32_t doc_idx) { + send_remove(bucket_idx, doc_idx, next_timestamp()); + } + + void send_update(uint32_t bucket_idx, uint32_t doc_idx, uint32_t timestamp) { + auto id = id_of(bucket_idx, doc_idx); + auto update = std::make_shared<document::DocumentUpdate>( + _node->getTestDocMan().getTypeRepo(), + _node->getTestDocMan().createRandomDocument()->getType(), id); + auto cmd = std::make_shared<api::UpdateCommand>(makeDocumentBucket({16, bucket_idx}), std::move(update), timestamp); + schedule_msg(cmd); + } + + void send_update(uint32_t bucket_idx, uint32_t doc_idx) { + send_update(bucket_idx, doc_idx, next_timestamp()); + } + + [[nodiscard]] api::Timestamp next_timestamp() { + auto ret = _next_timestamp; + ++_next_timestamp; + return ret; + } + + [[nodiscard]] vespalib::steady_time fake_now() const { + return _node->getClock().getMonotonicTime(); + } + + [[nodiscard]] vespalib::steady_time fake_deadline() const { + return _node->getClock().getMonotonicTime() + 60s; + } + + [[nodiscard]] FileStorHandler::LockedMessageBatch next_batch() { + return _handler->next_message_batch(0, fake_now(), fake_deadline()); + } + + template <typename CmdType> + static void assert_batch_msg_is(const FileStorHandler::LockedMessageBatch& batch, uint32_t msg_idx, + uint32_t expected_bucket_idx, uint32_t expected_doc_idx) + { + ASSERT_LT(msg_idx, batch.size()); + auto msg = batch.messages[msg_idx].first; + auto* as_cmd = dynamic_cast<const CmdType*>(msg.get()); + ASSERT_TRUE(as_cmd) << msg->toString() << " does not have the expected type"; + EXPECT_EQ(as_cmd->getBucketId(), BucketId(16, expected_bucket_idx)); + + auto id = as_cmd->getDocumentId(); + ASSERT_TRUE(id.getScheme().hasNumber()); + EXPECT_EQ(id.getScheme().getNumber(), expected_bucket_idx) << id; + std::string actual_id_part = id.getScheme().getNamespaceSpecific(); + std::string expected_id_part = std::to_string(expected_doc_idx); + EXPECT_EQ(actual_id_part, expected_id_part) << id; + } + + static void assert_batch_msg_is_put(const FileStorHandler::LockedMessageBatch& batch, uint32_t msg_idx, + uint32_t expected_bucket_idx, uint32_t expected_doc_idx) + { + assert_batch_msg_is<api::PutCommand>(batch, msg_idx, expected_bucket_idx, expected_doc_idx); + } + + static void assert_batch_msg_is_remove(const FileStorHandler::LockedMessageBatch& batch, uint32_t msg_idx, + uint32_t expected_bucket_idx, uint32_t expected_doc_idx) + { + assert_batch_msg_is<api::RemoveCommand>(batch, msg_idx, expected_bucket_idx, expected_doc_idx); + } + + static void assert_batch_msg_is_update(const FileStorHandler::LockedMessageBatch& batch, uint32_t msg_idx, + uint32_t expected_bucket_idx, uint32_t expected_doc_idx) + { + assert_batch_msg_is<api::UpdateCommand>(batch, msg_idx, expected_bucket_idx, expected_doc_idx); + } + + static void assert_batch_msg_is_get(const FileStorHandler::LockedMessageBatch& batch, uint32_t msg_idx, + uint32_t expected_bucket_idx, uint32_t expected_doc_idx) + { + assert_batch_msg_is<api::GetCommand>(batch, msg_idx, expected_bucket_idx, expected_doc_idx); + } + + enum Type { + Put, + Update, + Remove, + Get + }; + + static void assert_empty_batch(const FileStorHandler::LockedMessageBatch& batch) { + ASSERT_TRUE(batch.empty()); + ASSERT_FALSE(batch.lock); + } + + static void assert_batch(const FileStorHandler::LockedMessageBatch& batch, + uint32_t expected_bucket_idx, + std::initializer_list<std::pair<Type, uint32_t>> expected_msgs) + { + ASSERT_TRUE(batch.lock); + ASSERT_EQ(batch.lock->getBucket().getBucketId(), BucketId(16, expected_bucket_idx)); + ASSERT_EQ(batch.size(), expected_msgs.size()); + + uint32_t idx = 0; + for (const auto& msg : expected_msgs) { + switch (msg.first) { + case Type::Put: assert_batch_msg_is_put(batch, idx, expected_bucket_idx, msg.second); break; + case Type::Update: assert_batch_msg_is_update(batch, idx, expected_bucket_idx, msg.second); break; + case Type::Remove: assert_batch_msg_is_remove(batch, idx, expected_bucket_idx, msg.second); break; + case Type::Get: assert_batch_msg_is_get(batch, idx, expected_bucket_idx, msg.second); break; + default: FAIL(); + } + ++idx; + } + } +}; + +FeedOperationBatchingTest::FeedOperationBatchingTest() + : FileStorTestFixture(), + _top(), + _message_sender(), + _metrics(), + _handler(), + _next_timestamp(1000) +{ +} + +FeedOperationBatchingTest::~FeedOperationBatchingTest() = default; + +// Note: unless explicitly set by the testcase, max batch size is 3 + +TEST_F(FeedOperationBatchingTest, batching_is_disabled_with_1_max_batch_size) { + _handler->set_max_feed_op_batch_size(1); + send_puts({{1, 1}, {1, 2}, {2, 3}, {2, 4}}); + // No batching; has the same behavior as current FIFO + assert_batch(next_batch(), 1, {{Put, 1}}); + assert_batch(next_batch(), 1, {{Put, 2}}); + assert_batch(next_batch(), 2, {{Put, 3}}); + assert_batch(next_batch(), 2, {{Put, 4}}); + assert_empty_batch(next_batch()); +} + +TEST_F(FeedOperationBatchingTest, batching_is_limited_to_configured_max_size) { + send_puts({{1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}); + assert_batch(next_batch(), 1, {{Put, 1}, {Put, 2}, {Put, 3}}); + assert_batch(next_batch(), 1, {{Put, 4}, {Put, 5}}); + assert_empty_batch(next_batch()); +} + +TEST_F(FeedOperationBatchingTest, batching_can_consume_entire_queue) { + send_puts({{1, 1}, {1, 2}, {1, 3}}); + assert_batch(next_batch(), 1, {{Put, 1}, {Put, 2}, {Put, 3}}); + assert_empty_batch(next_batch()); +} + +TEST_F(FeedOperationBatchingTest, batching_is_only_done_for_single_bucket) { + send_puts({{1, 1}, {2, 2}, {2, 3}, {2, 4}, {3, 5}}); + assert_batch(next_batch(), 1, {{Put, 1}}); + assert_batch(next_batch(), 2, {{Put, 2}, {Put, 3}, {Put, 4}}); + assert_batch(next_batch(), 3, {{Put, 5}}); +} + +TEST_F(FeedOperationBatchingTest, batch_can_include_all_supported_feed_op_types) { + send_put(1, 1); + send_remove(1, 2); + send_update(1, 3); + assert_batch(next_batch(), 1, {{Put, 1}, {Remove, 2}, {Update, 3}}); +} + +TEST_F(FeedOperationBatchingTest, timed_out_reqeusts_are_ignored_by_batch) { + send_puts({{1, 1}}); + send_put(1, 2, next_timestamp(), 1s); + send_puts({{1, 3}}); + _node->getClock().addSecondsToTime(2); + // Put #2 with 1s timeout has expired in the queue and should not be returned as part of the batch + assert_batch(next_batch(), 1, {{Put, 1}, {Put, 3}}); + ASSERT_EQ(_top.getNumReplies(), 0); + // The actual timeout is handled by the next message fetch invocation + assert_empty_batch(next_batch()); + ASSERT_EQ(_top.getNumReplies(), 1); + EXPECT_EQ(dynamic_cast<api::StorageReply&>(*_top.getReply(0)).getResult().getResult(), api::ReturnCode::TIMEOUT); +} + +TEST_F(FeedOperationBatchingTest, non_feed_ops_are_not_batched) { + send_get(1, 2); + send_get(1, 3); + assert_batch(next_batch(), 1, {{Get, 2}}); + assert_batch(next_batch(), 1, {{Get, 3}}); +} + +TEST_F(FeedOperationBatchingTest, pipeline_stalled_by_non_feed_op) { + // It can reasonably be argued that we could batch _around_ a Get operation and still + // have correct behavior, but the Get here is just a stand-in for an arbitrary operation such + // as a Split (which changes the bucket set), which is rather more tricky to reason about. + // For simplicity and understandability, just stall the batch pipeline (at least for now). + send_get(1, 2); + send_puts({{1, 3}, {1, 4}}); + send_get(1, 5); + send_puts({{1, 6}, {1, 7}}); + + assert_batch(next_batch(), 1, {{Get, 2}}); // If first op is non-feed, only it should be returned + assert_batch(next_batch(), 1, {{Put, 3}, {Put, 4}}); + assert_batch(next_batch(), 1, {{Get, 5}}); + assert_batch(next_batch(), 1, {{Put, 6}, {Put, 7}}); +} + +TEST_F(FeedOperationBatchingTest, pipeline_stalled_by_concurrent_ops_to_same_document) { + // 2 ops to doc #2. Since this is expected to be a very rare edge case, just + // stop batching at that point and defer the concurrent op to the next batch. + send_puts({{1, 1}, {1, 2}, {1, 3}, {1, 2}, {1, 4}}); + assert_batch(next_batch(), 1, {{Put, 1}, {Put, 2}, {Put, 3}}); + assert_batch(next_batch(), 1, {{Put, 2}, {Put, 4}}); +} + +TEST_F(FeedOperationBatchingTest, batch_respects_persistence_throttling) { + vespalib::SharedOperationThrottler::DynamicThrottleParams params; + params.min_window_size = 3; + params.max_window_size = 3; + params.window_size_increment = 1; + _handler->use_dynamic_operation_throttling(true); + _handler->reconfigure_dynamic_throttler(params); + _handler->set_max_feed_op_batch_size(10); // > win size to make sure we test the right thing + + send_puts({{1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}}); + auto batch = next_batch(); // holds 3 throttle tokens + assert_batch(batch, 1, {{Put, 1}, {Put, 2}, {Put, 3}}); + // No more throttle tokens available + assert_empty_batch(next_batch()); +} + +} // storage diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 1ccd51d3f06..f12b85eb2ea 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -43,6 +43,7 @@ LOG_SETUP(".filestormanagertest"); using std::unique_ptr; using document::Document; +using document::BucketId; using namespace storage::api; using storage::spi::test::makeSpiBucket; using document::test::makeDocumentBucket; @@ -405,6 +406,38 @@ TEST_F(FileStorManagerTest, put) { } } +TEST_F(FileStorManagerTest, feed_op_batch_updates_bucket_db_and_reply_bucket_info) { + PersistenceHandlerComponents c(*this); + c.filestorHandler->set_max_feed_op_batch_size(10); + BucketId bucket_id(16, 1); + createBucket(bucket_id); + constexpr uint32_t n = 10; + // No persistence thread started yet, so no chance of racing + for (uint32_t i = 0; i < n; ++i) { + auto put = make_put_command(120, vespalib::make_string("id:foo:testdoctype1:n=1:%u", i), Timestamp(1000) + i); + put->setAddress(_storage3); + c.filestorHandler->schedule(put); + } + auto pt = c.make_disk_thread(); + c.filestorHandler->flush(true); + c.top.waitForMessages(n, _waitTime); + c.executor.sync_all(); // Ensure all async reply processing tasks must have completed. + api::BucketInfo expected_bucket_info; + { + StorBucketDatabase::WrappedEntry entry(_node->getStorageBucketDatabase().get(bucket_id, "foo")); + ASSERT_TRUE(entry.exists()); + EXPECT_EQ(entry->getBucketInfo().getDocumentCount(), n); + expected_bucket_info = entry->getBucketInfo(); + } + // All replies should have the _same_ bucket info due to being processed in the same batch. + auto replies = c.top.getRepliesOnce(); + for (auto& reply : replies) { + auto actual_bucket_info = dynamic_cast<api::PutReply&>(*reply).getBucketInfo(); + EXPECT_EQ(actual_bucket_info, expected_bucket_info); + } + c.filestorHandler->close(); // Ensure persistence thread is no longer in message fetch code +} + TEST_F(FileStorManagerTest, running_task_against_unknown_bucket_fails) { TestFileStorComponents c(*this); @@ -726,7 +759,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { filestorHandler.schedule(cmd); } - std::this_thread::sleep_for(51ms); + _node->getClock().addMilliSecondsToTime(51); for (;;) { auto lock = filestorHandler.getNextMessage(stripeId); if (lock.lock.get()) { diff --git a/storage/src/tests/persistence/filestorage/gtest_runner.cpp b/storage/src/tests/persistence/filestorage/gtest_runner.cpp index 5d1fde4130c..1ed7bc91843 100644 --- a/storage/src/tests/persistence/filestorage/gtest_runner.cpp +++ b/storage/src/tests/persistence/filestorage/gtest_runner.cpp @@ -1,8 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/gtest/gtest.h> +#include <absl/debugging/failure_signal_handler.h> #include <vespa/log/log.h> LOG_SETUP("storage_filestorage_gtest_runner"); -GTEST_MAIN_RUN_ALL_TESTS() +int main(int argc, char* argv[]) { + absl::FailureSignalHandlerOptions opts; + opts.call_previous_handler = true; + opts.use_alternate_stack = false; // Suboptimal, but needed to get proper backtracing (for some reason...) + absl::InstallFailureSignalHandler(opts); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 59e0853cc21..ab176ebb9cb 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -143,7 +143,7 @@ AsyncHandler::AsyncHandler(const PersistenceUtil & env, spi::PersistenceProvider MessageTracker::UP AsyncHandler::handleRunTask(RunTaskCommand& cmd, MessageTracker::UP tracker) const { auto task = makeResultTask([tracker = std::move(tracker)](spi::Result::UP response) { - tracker->checkForError(*response); + (void)tracker->checkForError(*response); tracker->sendReply(); }); spi::Bucket bucket(cmd.getBucket()); @@ -169,7 +169,7 @@ AsyncHandler::handlePut(api::PutCommand& cmd, MessageTracker::UP trackerUP) cons spi::Bucket bucket = _env.getBucket(cmd.getDocumentId(), cmd.getBucket()); auto task = makeResultTask([tracker = std::move(trackerUP)](spi::Result::UP response) { - tracker->checkForError(*response); + (void)tracker->checkForError(*response); tracker->sendReply(); }); _spi.putAsync(bucket, spi::Timestamp(cmd.getTimestamp()), cmd.getDocument(), @@ -517,7 +517,7 @@ AsyncHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, MessageTrack } auto task = makeResultTask([&cmd, tracker = std::move(tracker), removed = to_remove.size()](spi::Result::UP response) { - tracker->checkForError(*response); + (void)tracker->checkForError(*response); tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, removed)); tracker->sendReply(); }); diff --git a/storage/src/vespa/storage/persistence/batched_message.h b/storage/src/vespa/storage/persistence/batched_message.h new file mode 100644 index 00000000000..c23383edfde --- /dev/null +++ b/storage/src/vespa/storage/persistence/batched_message.h @@ -0,0 +1,14 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.// +#pragma once + +#include "shared_operation_throttler.h" +#include <memory> +#include <utility> + +namespace storage { + +namespace api { class StorageMessage; } + +using BatchedMessage = std::pair<std::shared_ptr<api::StorageMessage>, ThrottleToken>; + +} diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index a89b705de1b..274d59899d9 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -5,4 +5,15 @@ namespace storage { FileStorHandler::LockedMessage::~LockedMessage() = default; +FileStorHandler::LockedMessageBatch::LockedMessageBatch(LockedMessage&& initial_msg) + : lock(std::move(initial_msg.lock)), + messages() +{ + if (lock) { + messages.emplace_back(std::move(initial_msg.msg), std::move(initial_msg.throttle_token)); + } +} + +FileStorHandler::LockedMessageBatch::~LockedMessageBatch() = default; + } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index 68b8411c762..6a8b74baf1d 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -1,23 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * \class storage::FileStorHandler - * \ingroup storage - * - * \brief Common resource for filestor threads - * - * Takes care of the interface between file stor threads and the file stor - * manager to avoid circular dependencies, and confine the implementation that - * needs to worry about locking between these components. - */ - #pragma once #include <vespa/document/bucket/bucket.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/storage/persistence/batched_message.h> #include <vespa/storage/persistence/shared_operation_throttler.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/vespalib/util/small_vector.h> namespace storage { namespace api { @@ -80,13 +70,7 @@ public: std::shared_ptr<api::StorageMessage> msg; ThrottleToken throttle_token; - LockedMessage() noexcept = default; - LockedMessage(std::shared_ptr<BucketLockInterface> lock_, - std::shared_ptr<api::StorageMessage> msg_) noexcept - : lock(std::move(lock_)), - msg(std::move(msg_)), - throttle_token() - {} + constexpr LockedMessage() noexcept = default; LockedMessage(std::shared_ptr<BucketLockInterface> lock_, std::shared_ptr<api::StorageMessage> msg_, ThrottleToken token) noexcept @@ -98,27 +82,40 @@ public: ~LockedMessage(); }; + struct LockedMessageBatch { + std::shared_ptr<BucketLockInterface> lock; + vespalib::SmallVector<BatchedMessage, 1> messages; + + LockedMessageBatch() = default; + explicit LockedMessageBatch(LockedMessage&& initial_msg); + LockedMessageBatch(LockedMessageBatch&&) noexcept = default; + ~LockedMessageBatch(); + + [[nodiscard]] bool empty() const noexcept { return messages.empty(); } + [[nodiscard]] size_t size() const noexcept { return messages.size(); } + // Precondition: messages.size() == 1 + [[nodiscard]] LockedMessage release_as_single_msg() noexcept; + }; + class ScheduleAsyncResult { - private: - bool _was_scheduled; + bool _was_scheduled; LockedMessage _async_message; - public: - ScheduleAsyncResult() : _was_scheduled(false), _async_message() {} - explicit ScheduleAsyncResult(LockedMessage&& async_message_in) + constexpr ScheduleAsyncResult() noexcept : _was_scheduled(false), _async_message() {} + explicit ScheduleAsyncResult(LockedMessage&& async_message_in) noexcept : _was_scheduled(true), _async_message(std::move(async_message_in)) {} - bool was_scheduled() const { + [[nodiscard]] bool was_scheduled() const noexcept { return _was_scheduled; } - bool has_async_message() const { - return _async_message.lock.get() != nullptr; + [[nodiscard]] bool has_async_message() const noexcept { + return static_cast<bool>(_async_message.lock); } - const LockedMessage& async_message() const { + [[nodiscard]] const LockedMessage& async_message() const noexcept { return _async_message; } - LockedMessage&& release_async_message() { + [[nodiscard]] LockedMessage&& release_async_message() noexcept { return std::move(_async_message); } }; @@ -129,7 +126,7 @@ public: }; FileStorHandler() : _getNextMessageTimout(100ms) { } - virtual ~FileStorHandler() = default; + ~FileStorHandler() override = default; /** @@ -171,7 +168,9 @@ public: * * @param stripe The stripe to get messages for */ - virtual LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) = 0; + [[nodiscard]] virtual LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) = 0; + + [[nodiscard]] virtual LockedMessageBatch next_message_batch(uint32_t stripe, vespalib::steady_time now, vespalib::steady_time deadline) = 0; /** Only used for testing, should be removed */ LockedMessage getNextMessage(uint32_t stripeId) { @@ -189,8 +188,6 @@ public: * NB: As current operation can be a split or join operation, make sure that * you always wait for current to finish, if is a super or sub bucket of * the bucket we're locking. - * - * */ virtual BucketLockInterface::SP lock(const document::Bucket&, api::LockingRequirements lockReq) = 0; @@ -287,6 +284,8 @@ public: virtual void use_dynamic_operation_throttling(bool use_dynamic) noexcept = 0; virtual void set_throttle_apply_bucket_diff_ops(bool throttle_apply_bucket_diff) noexcept = 0; + + virtual void set_max_feed_op_batch_size(uint32_t max_batch) noexcept = 0; private: vespalib::duration _getNextMessageTimout; }; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 1984d44652a..7589fb3cdb3 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -12,6 +12,7 @@ #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/string_escape.h> @@ -60,7 +61,8 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe _max_active_merges_per_stripe(per_stripe_merge_limit(numThreads, numStripes)), _paused(false), _throttle_apply_bucket_diff_ops(false), - _last_active_operations_stats() + _last_active_operations_stats(), + _max_feed_op_batch_size(1) { assert(numStripes > 0); _stripes.reserve(numStripes); @@ -241,8 +243,7 @@ FileStorHandlerImpl::schedule(const std::shared_ptr<api::StorageMessage>& msg) { if (getState() == FileStorHandler::AVAILABLE) { document::Bucket bucket = getStorageMessageBucket(*msg); - stripe(bucket).schedule(MessageEntry(msg, bucket)); - return true; + return stripe(bucket).schedule(MessageEntry(msg, bucket, _component.getClock().getMonotonicTime())); } return false; } @@ -252,7 +253,7 @@ FileStorHandlerImpl::schedule_and_get_next_async_message(const std::shared_ptr<a { if (getState() == FileStorHandler::AVAILABLE) { document::Bucket bucket = getStorageMessageBucket(*msg); - return ScheduleAsyncResult(stripe(bucket).schedule_and_get_next_async_message(MessageEntry(msg, bucket))); + return ScheduleAsyncResult(stripe(bucket).schedule_and_get_next_async_message(MessageEntry(msg, bucket, _component.getClock().getMonotonicTime()))); } return {}; } @@ -403,10 +404,25 @@ FileStorHandlerImpl::getNextMessage(uint32_t stripeId, vespalib::steady_time dea if (!tryHandlePause()) { return {}; // Still paused, return to allow tick. } - return _stripes[stripeId].getNextMessage(deadline); } +FileStorHandler::LockedMessageBatch +FileStorHandlerImpl::next_message_batch(uint32_t stripe_id, vespalib::steady_time now, vespalib::steady_time deadline) +{ + if (!tryHandlePause()) { + return {}; + } + return _stripes[stripe_id].next_message_batch(now, deadline); +} + +FileStorHandler::LockedMessage +FileStorHandlerImpl::LockedMessageBatch::release_as_single_msg() noexcept +{ + assert(lock && messages.size() == 1); + return {std::move(lock), std::move(messages[0].first), std::move(messages[0].second)}; +} + std::shared_ptr<FileStorHandler::BucketLockInterface> FileStorHandlerImpl::Stripe::lock(const document::Bucket &bucket, api::LockingRequirements lockReq) { std::unique_lock guard(*_lock); @@ -858,9 +874,10 @@ FileStorHandlerImpl::sendReplyDirectly(const std::shared_ptr<api::StorageReply>& } FileStorHandlerImpl::MessageEntry::MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, - const document::Bucket &bucket) + const document::Bucket& bucket, + vespalib::steady_time scheduled_at_time) : _command(cmd), - _timer(), + _timer(scheduled_at_time), _bucket(bucket), _priority(cmd->getPriority()) { } @@ -939,9 +956,8 @@ FileStorHandlerImpl::Stripe::operation_type_should_be_throttled(api::MessageType } FileStorHandler::LockedMessage -FileStorHandlerImpl::Stripe::getNextMessage(vespalib::steady_time deadline) +FileStorHandlerImpl::Stripe::next_message_impl(monitor_guard& guard, vespalib::steady_time deadline) { - std::unique_lock guard(*_lock); ThrottleToken throttle_token; // Try to grab a message+lock, immediately retrying once after a wait // if none can be found and then exiting if the same is the case on the @@ -993,6 +1009,93 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::steady_time deadline) } FileStorHandler::LockedMessage +FileStorHandlerImpl::Stripe::getNextMessage(vespalib::steady_time deadline) +{ + std::unique_lock guard(*_lock); + return next_message_impl(guard, deadline); +} + +namespace { + +constexpr bool is_batchable_feed_op(api::MessageType::Id id) noexcept { + return (id == api::MessageType::PUT_ID || + id == api::MessageType::REMOVE_ID || + id == api::MessageType::UPDATE_ID); +} + +// Precondition: msg must be a feed operation request (put, remove, update) +document::GlobalId gid_from_feed_op(const api::StorageMessage& msg) { + switch (msg.getType().getId()) { + case api::MessageType::PUT_ID: + return static_cast<const api::PutCommand&>(msg).getDocumentId().getGlobalId(); + case api::MessageType::REMOVE_ID: + return static_cast<const api::RemoveCommand&>(msg).getDocumentId().getGlobalId(); + case api::MessageType::UPDATE_ID: + return static_cast<const api::UpdateCommand&>(msg).getDocumentId().getGlobalId(); + default: abort(); + } +} + +} // anon ns + +FileStorHandler::LockedMessageBatch +FileStorHandlerImpl::Stripe::next_message_batch(vespalib::steady_time now, vespalib::steady_time deadline) +{ + const auto max_batch_size = _owner.max_feed_op_batch_size(); + + std::unique_lock guard(*_lock); + auto initial_locked = next_message_impl(guard, deadline); + if (!initial_locked.lock || !is_batchable_feed_op(initial_locked.msg->getType().getId()) || (max_batch_size == 1)) { + return LockedMessageBatch(std::move(initial_locked)); + } + LockedMessageBatch batch(std::move(initial_locked)); + fill_feed_op_batch(guard, batch, max_batch_size, now); + return batch; +} + +void +FileStorHandlerImpl::Stripe::fill_feed_op_batch(monitor_guard& guard, LockedMessageBatch& batch, + uint32_t max_batch_size, vespalib::steady_time now) +{ + assert(batch.size() == 1); + assert(guard.owns_lock()); + BucketIdx& idx = bmi::get<2>(*_queue); + auto bucket_msgs = idx.equal_range(batch.lock->getBucket()); + // Process in FIFO order (_not_ priority order) until we hit the end, a non-batchable operation + // (implicit pipeline stall since bucket set might change) or can't get another throttle token. + // We also stall the pipeline if we get a concurrent modification to the same document (not expected, + // as the distributors should prevent this, but _technically_ it is possible). + const auto expected_max_size = std::min(ssize_t(max_batch_size), std::distance(bucket_msgs.first, bucket_msgs.second) + 1); + vespalib::hash_set<document::GlobalId, document::GlobalId::hash> gids_in_batch(expected_max_size); + gids_in_batch.insert(gid_from_feed_op(*batch.messages[0].first)); + for (auto it = bucket_msgs.first; (it != bucket_msgs.second) && (batch.messages.size() < max_batch_size);) { + if (!is_batchable_feed_op(it->_command->getType().getId())) { + break; + } + auto [existing_iter, inserted] = gids_in_batch.insert(gid_from_feed_op(*it->_command)); + if (!inserted) { + break; // Already present in batch + } + if (messageTimedOutInQueue(*it->_command, now - it->_timer.start_time())) { + // We just ignore timed out ops here; actually generating a timeout reply will be done by + // next_message_impl() during a subsequent invocation. This avoids having to deal with any + // potential issues caused by sending a reply up while holding the queue lock, since we + // can't release it here. + ++it; + continue; + } + auto throttle_token = _owner.operation_throttler().try_acquire_one(); + if (!throttle_token.valid()) { + break; + } + // Note: iterator is const; can't std::move(it->_command) + batch.messages.emplace_back(it->_command, std::move(throttle_token)); + it = idx.erase(it); + } + update_cached_queue_size(guard); +} + +FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard) { if (_owner.isPaused()) { @@ -1021,9 +1124,11 @@ FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter, ThrottleToken throttle_token) { - std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop(_metrics->averageQueueWaitingTime))); + std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop( + _owner._component.getClock().getMonotonicTime(), + _metrics->averageQueueWaitingTime))); - std::shared_ptr<api::StorageMessage> msg = std::move(iter->_command); + std::shared_ptr<api::StorageMessage> msg = iter->_command; // iter is const; can't std::move() document::Bucket bucket(iter->_bucket); idx.erase(iter); // iter not used after this point. update_cached_queue_size(guard); @@ -1032,7 +1137,6 @@ FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx auto locker = std::make_unique<BucketLock>(guard, *this, bucket, msg->getPriority(), msg->getType().getId(), msg->getMsgId(), msg->lockingRequirements()); - guard.unlock(); return {std::move(locker), std::move(msg), std::move(throttle_token)}; } else { std::shared_ptr<api::StorageReply> msgReply(makeQueueTimeoutReply(*msg)); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index f7c9b218779..ac8e5c52cbf 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -55,7 +55,9 @@ public: document::Bucket _bucket; uint8_t _priority; - MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, const document::Bucket &bId); + MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, + const document::Bucket& bucket, + vespalib::steady_time scheduled_at_time); MessageEntry(MessageEntry &&) noexcept ; MessageEntry(const MessageEntry &) noexcept; MessageEntry & operator = (const MessageEntry &) = delete; @@ -66,13 +68,13 @@ public: } }; - using PriorityOrder = bmi::ordered_non_unique<bmi::identity<MessageEntry> >; - using BucketOrder = bmi::ordered_non_unique<bmi::member<MessageEntry, document::Bucket, &MessageEntry::_bucket>>; - + // ordered_non_unique shall preserve insertion order as iteration order of equal keys, but this is rather magical... + using PriorityOrder = bmi::ordered_non_unique<bmi::identity<MessageEntry>>; + using BucketOrder = bmi::ordered_non_unique<bmi::member<MessageEntry, document::Bucket, &MessageEntry::_bucket>>; using PriorityQueue = bmi::multi_index_container<MessageEntry, bmi::indexed_by<bmi::sequenced<>, PriorityOrder, BucketOrder>>; + using PriorityIdx = bmi::nth_index<PriorityQueue, 1>::type; + using BucketIdx = bmi::nth_index<PriorityQueue, 2>::type; - using PriorityIdx = bmi::nth_index<PriorityQueue, 1>::type; - using BucketIdx = bmi::nth_index<PriorityQueue, 2>::type; using Clock = std::chrono::steady_clock; using monitor_guard = std::unique_lock<std::mutex>; using atomic_size_t = vespalib::datastore::AtomicValueWrapper<size_t>; @@ -114,7 +116,7 @@ public: ActiveOperationsStats &stats() { return _stats; } }; SafeActiveOperationsStats() : _lock(std::make_unique<std::mutex>()), _stats() {} - Guard guard() { return Guard(*_lock, _stats, ctor_tag()); } + [[nodiscard]] Guard guard() { return Guard(*_lock, _stats, ctor_tag()); } }; Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender); @@ -123,8 +125,8 @@ public: Stripe & operator =(const Stripe &) = delete; ~Stripe(); void flush(); - bool schedule(MessageEntry messageEntry); - FileStorHandler::LockedMessage schedule_and_get_next_async_message(MessageEntry entry); + [[nodiscard]] bool schedule(MessageEntry messageEntry); + [[nodiscard]] FileStorHandler::LockedMessage schedule_and_get_next_async_message(MessageEntry entry); void waitUntilNoLocks() const; void abort(std::vector<std::shared_ptr<api::StorageReply>> & aborted, const AbortBucketOperationsCommand& cmd); void waitInactive(const AbortBucketOperationsCommand& cmd) const; @@ -151,18 +153,19 @@ public: api::LockingRequirements lockReq, bool count_as_active_merge, const LockEntry & lockEntry); - std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket, api::LockingRequirements lockReq); + [[nodiscard]] std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket, api::LockingRequirements lockReq); void failOperations(const document::Bucket & bucket, const api::ReturnCode & code); - FileStorHandler::LockedMessage getNextMessage(vespalib::steady_time deadline); + [[nodiscard]] FileStorHandler::LockedMessage getNextMessage(vespalib::steady_time deadline); + [[nodiscard]] FileStorHandler::LockedMessageBatch next_message_batch(vespalib::steady_time now, vespalib::steady_time deadline); void dumpQueue(std::ostream & os) const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; - std::mutex & exposeLock() { return *_lock; } - PriorityQueue & exposeQueue() { return *_queue; } - BucketIdx & exposeBucketIdx() { return bmi::get<2>(*_queue); } + [[nodiscard]] std::mutex & exposeLock() { return *_lock; } + [[nodiscard]] PriorityQueue & exposeQueue() { return *_queue; } + [[nodiscard]] BucketIdx & exposeBucketIdx() { return bmi::get<2>(*_queue); } void setMetrics(FileStorStripeMetrics * metrics) { _metrics = metrics; } - ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const; + [[nodiscard]] ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const; private: void update_cached_queue_size(const std::lock_guard<std::mutex> &) { _cached_queue_size.store_relaxed(_queue->size()); @@ -170,15 +173,20 @@ public: void update_cached_queue_size(const std::unique_lock<std::mutex> &) { _cached_queue_size.store_relaxed(_queue->size()); } - bool hasActive(monitor_guard & monitor, const AbortBucketOperationsCommand& cmd) const; - FileStorHandler::LockedMessage get_next_async_message(monitor_guard& guard); + [[nodiscard]] bool hasActive(monitor_guard & monitor, const AbortBucketOperationsCommand& cmd) const; + [[nodiscard]] FileStorHandler::LockedMessage get_next_async_message(monitor_guard& guard); [[nodiscard]] bool operation_type_should_be_throttled(api::MessageType::Id type_id) const noexcept; + [[nodiscard]] FileStorHandler::LockedMessage next_message_impl(monitor_guard& held_lock, + vespalib::steady_time deadline); + void fill_feed_op_batch(monitor_guard& held_lock, LockedMessageBatch& batch, + uint32_t max_batch_size, vespalib::steady_time now); + // Precondition: the bucket used by `iter`s operation is not locked in a way that conflicts // with its locking requirements. - FileStorHandler::LockedMessage getMessage(monitor_guard & guard, PriorityIdx & idx, - PriorityIdx::iterator iter, - ThrottleToken throttle_token); + [[nodiscard]] FileStorHandler::LockedMessage getMessage(monitor_guard & guard, PriorityIdx & idx, + PriorityIdx::iterator iter, + ThrottleToken throttle_token); using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>; const FileStorHandlerImpl &_owner; MessageSender &_messageSender; @@ -233,7 +241,8 @@ public: bool schedule(const std::shared_ptr<api::StorageMessage>&) override; ScheduleAsyncResult schedule_and_get_next_async_message(const std::shared_ptr<api::StorageMessage>& msg) override; - FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) override; + LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) override; + LockedMessageBatch next_message_batch(uint32_t stripe, vespalib::steady_time now, vespalib::steady_time deadline) override; void remapQueueAfterJoin(const RemapInfo& source, RemapInfo& target) override; void remapQueueAfterSplit(const RemapInfo& source, RemapInfo& target1, RemapInfo& target2) override; @@ -292,6 +301,13 @@ public: _throttle_apply_bucket_diff_ops.store(throttle_apply_bucket_diff, std::memory_order_relaxed); } + void set_max_feed_op_batch_size(uint32_t max_batch) noexcept override { + _max_feed_op_batch_size.store(max_batch, std::memory_order_relaxed); + } + [[nodiscard]] uint32_t max_feed_op_batch_size() const noexcept { + return _max_feed_op_batch_size.load(std::memory_order_relaxed); + } + // Implements ResumeGuard::Callback void resume() override; @@ -316,6 +332,7 @@ private: std::atomic<bool> _paused; std::atomic<bool> _throttle_apply_bucket_diff_ops; std::optional<ActiveOperationsStats> _last_active_operations_stats; + std::atomic<uint32_t> _max_feed_op_batch_size; // Returns the index in the targets array we are sending to, or -1 if none of them match. int calculateTargetBasedOnDocId(const api::StorageMessage& msg, std::vector<RemapInfo*>& targets); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 0046bd96b65..23de39f7130 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -239,6 +239,7 @@ FileStorManager::on_configure(const StorFilestorConfig& config) auto updated_dyn_throttle_params = dynamic_throttle_params_from_config(config, _threads.size()); _filestorHandler->reconfigure_dynamic_throttler(updated_dyn_throttle_params); } + _filestorHandler->set_max_feed_op_batch_size(std::max(1, config.maxFeedOpBatchSize)); // TODO remove once desired throttling behavior is set in stone { _filestorHandler->use_dynamic_operation_throttling(use_dynamic_throttling); diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 0612798b43a..a01a4656d01 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -134,7 +134,7 @@ PersistenceHandler::processMessage(api::StorageMessage& msg, MessageTracker::UP _env._metrics.operations.inc(); if (msg.getType().isReply()) { - try{ + try { LOG(debug, "Handling reply: %s", msg.toString().c_str()); LOG(spam, "Message content: %s", msg.toString(true).c_str()); return handleReply(static_cast<api::StorageReply&>(msg), std::move(tracker)); @@ -161,9 +161,7 @@ PersistenceHandler::processMessage(api::StorageMessage& msg, MessageTracker::UP void PersistenceHandler::processLockedMessage(FileStorHandler::LockedMessage lock) const { - LOG(debug, "NodeIndex %d, ptr=%p", _env._nodeIndex, lock.msg.get()); api::StorageMessage & msg(*lock.msg); - // Important: we _copy_ the message shared_ptr instead of moving to ensure that `msg` remains // valid even if the tracker is destroyed by an exception in processMessage(). auto tracker = std::make_unique<MessageTracker>(framework::MilliSecTimer(_clock), _env, _env._fileStorHandler, @@ -174,4 +172,24 @@ PersistenceHandler::processLockedMessage(FileStorHandler::LockedMessage lock) co } } +void +PersistenceHandler::process_locked_message_batch(std::shared_ptr<FileStorHandler::BucketLockInterface> lock, + std::span<BatchedMessage> bucket_messages) +{ + const auto bucket = lock->getBucket(); + auto batch = std::make_shared<AsyncMessageBatch>(std::move(lock), _env, _env._fileStorHandler); + for (auto& bm : bucket_messages) { + assert(bm.first->getBucket() == bucket); + // Important: we _copy_ the message shared_ptr instead of moving to ensure that `*bm.first` remains + // valid even if the tracker is destroyed by an exception in processMessage(). All std::exceptions + // are caught there, so we do not expect our loop to be interrupted. + auto tracker = std::make_unique<MessageTracker>(framework::MilliSecTimer(_clock), _env, batch, + batch->deferred_sender_stub(), bm.first, std::move(bm.second)); + tracker = processMessage(*bm.first, std::move(tracker)); + if (tracker) { + tracker->sendReply(); // Actually defers to batch reply queue + } + } +} + } diff --git a/storage/src/vespa/storage/persistence/persistencehandler.h b/storage/src/vespa/storage/persistence/persistencehandler.h index 0da518a1cfa..224bb70a16b 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.h +++ b/storage/src/vespa/storage/persistence/persistencehandler.h @@ -2,14 +2,16 @@ #pragma once -#include "processallhandler.h" -#include "mergehandler.h" #include "asynchandler.h" +#include "batched_message.h" +#include "mergehandler.h" #include "persistenceutil.h" -#include "splitjoinhandler.h" +#include "processallhandler.h" #include "simplemessagehandler.h" +#include "splitjoinhandler.h" #include <vespa/storage/common/storagecomponent.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> +#include <span> namespace storage { @@ -29,6 +31,8 @@ public: ~PersistenceHandler(); void processLockedMessage(FileStorHandler::LockedMessage lock) const; + void process_locked_message_batch(std::shared_ptr<FileStorHandler::BucketLockInterface> lock, + std::span<BatchedMessage> bucket_messages); //TODO Rewrite tests to avoid this api leak const AsyncHandler & asyncHandler() const { return _asyncHandler; } diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index a98418281d2..5710c3fe26f 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -14,6 +14,7 @@ PersistenceThread::PersistenceThread(PersistenceHandler & persistenceHandler, Fi uint32_t stripeId, framework::Component & component) : _persistenceHandler(persistenceHandler), _fileStorHandler(fileStorHandler), + _clock(component.getClock()), _stripeId(stripeId), _thread() { @@ -36,14 +37,19 @@ PersistenceThread::run(framework::ThreadHandle& thread) vespalib::duration max_wait_time = vespalib::adjustTimeoutByDetectedHz(100ms); while (!thread.interrupted()) { - vespalib::steady_time now = vespalib::steady_clock::now(); + vespalib::steady_time now = _clock.getMonotonicTime(); thread.registerTick(framework::UNKNOWN_CYCLE, now); vespalib::steady_time deadline = now + max_wait_time; - FileStorHandler::LockedMessage lock(_fileStorHandler.getNextMessage(_stripeId, deadline)); - - if (lock.lock) { - _persistenceHandler.processLockedMessage(std::move(lock)); + auto batch = _fileStorHandler.next_message_batch(_stripeId, now, deadline); + if (!batch.empty()) { + // Special-case single message batches, as actually scheduling a full batch has more + // overhead due to extra bookkeeping state and deferred reply-sending. + if (batch.size() == 1) { + _persistenceHandler.processLockedMessage(batch.release_as_single_msg()); + } else { + _persistenceHandler.process_locked_message_batch(std::move(batch.lock), batch.messages); + } } } LOG(debug, "Closing down persistence thread"); diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index aacd1dd4830..2e9852ada73 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -8,6 +8,7 @@ namespace storage { namespace framework { + struct Clock; class Component; class Thread; } @@ -27,10 +28,11 @@ public: framework::Thread& getThread() override { return *_thread; } private: - PersistenceHandler & _persistenceHandler; - FileStorHandler & _fileStorHandler; - uint32_t _stripeId; - std::unique_ptr<framework::Thread> _thread; + PersistenceHandler& _persistenceHandler; + FileStorHandler& _fileStorHandler; + const framework::Clock& _clock; + uint32_t _stripeId; + std::unique_ptr<framework::Thread> _thread; void run(framework::ThreadHandle&) override; }; diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index c975721c855..febc494aff1 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -11,41 +11,83 @@ LOG_SETUP(".persistence.util"); namespace storage { namespace { - bool isBatchable(api::MessageType::Id id) - { - return (id == api::MessageType::PUT_ID || - id == api::MessageType::REMOVE_ID || - id == api::MessageType::UPDATE_ID); - } - bool hasBucketInfo(api::MessageType::Id id) - { - return (isBatchable(id) || - (id == api::MessageType::REMOVELOCATION_ID || - id == api::MessageType::JOINBUCKETS_ID)); +constexpr bool is_batchable(api::MessageType::Id id) noexcept { + return (id == api::MessageType::PUT_ID || + id == api::MessageType::REMOVE_ID || + id == api::MessageType::UPDATE_ID); +} + +constexpr bool has_bucket_info(api::MessageType::Id id) noexcept { + return (is_batchable(id) || + (id == api::MessageType::REMOVELOCATION_ID || + id == api::MessageType::JOINBUCKETS_ID)); +} + +constexpr vespalib::duration WARN_ON_SLOW_OPERATIONS = 5s; + +} // anon ns + +DeferredReplySenderStub::DeferredReplySenderStub() = default; +DeferredReplySenderStub::~DeferredReplySenderStub() = default; + +AsyncMessageBatch::AsyncMessageBatch(std::shared_ptr<FileStorHandler::BucketLockInterface> bucket_lock, + const PersistenceUtil& env, + MessageSender& reply_sender) noexcept + : _bucket_lock(std::move(bucket_lock)), + _env(env), + _reply_sender(reply_sender), + _deferred_sender_stub() +{ + assert(_bucket_lock); +} + +AsyncMessageBatch::~AsyncMessageBatch() { + const auto bucket_info = _env.getBucketInfo(_bucket_lock->getBucket()); + _env.updateBucketDatabase(_bucket_lock->getBucket(), bucket_info); + + std::lock_guard lock(_deferred_sender_stub._mutex); // Ensure visibility of posted replies + for (auto& reply : _deferred_sender_stub._deferred_replies) { + if (reply->getResult().success()) { + dynamic_cast<api::BucketInfoReply&>(*reply).setBucketInfo(bucket_info); + } + _reply_sender.sendReplyDirectly(reply); } - const vespalib::duration WARN_ON_SLOW_OPERATIONS = 5s; + LOG(debug, "Processed async feed message batch of %zu ops for %s. New bucket info is %s", + _deferred_sender_stub._deferred_replies.size(), + _bucket_lock->getBucket().toString().c_str(), bucket_info.toString().c_str()); } MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, FileStorHandler::BucketLockInterface::SP bucketLock, - api::StorageMessage::SP msg, + std::shared_ptr<api::StorageMessage> msg, ThrottleToken throttle_token) - : MessageTracker(timer, env, replySender, true, std::move(bucketLock), std::move(msg), std::move(throttle_token)) + : MessageTracker(timer, env, replySender, true, std::move(bucketLock), {}, std::move(msg), std::move(throttle_token)) +{} + +MessageTracker::MessageTracker(const framework::MilliSecTimer& timer, + const PersistenceUtil& env, + std::shared_ptr<AsyncMessageBatch> batch, + MessageSender& deferred_reply_sender, + std::shared_ptr<api::StorageMessage> msg, + ThrottleToken throttle_token) + : MessageTracker(timer, env, deferred_reply_sender, false, {}, std::move(batch), std::move(msg), std::move(throttle_token)) {} MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, - bool updateBucketInfo, - FileStorHandler::BucketLockInterface::SP bucketLock, - api::StorageMessage::SP msg, + bool update_bucket_info, + std::shared_ptr<FileStorHandler::BucketLockInterface> bucket_lock, + std::shared_ptr<AsyncMessageBatch> part_of_batch, + std::shared_ptr<api::StorageMessage> msg, ThrottleToken throttle_token) : _sendReply(true), - _updateBucketInfo(updateBucketInfo && hasBucketInfo(msg->getType().getId())), - _bucketLock(std::move(bucketLock)), + _updateBucketInfo(update_bucket_info && has_bucket_info(msg->getType().getId())), + _bucketLock(std::move(bucket_lock)), + _part_of_batch(std::move(part_of_batch)), _msg(std::move(msg)), _throttle_token(std::move(throttle_token)), _context(_msg->getPriority(), _msg->getTrace().getLevel()), @@ -61,7 +103,7 @@ MessageTracker::createForTesting(const framework::MilliSecTimer & timer, Persist FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg) { return MessageTracker::UP(new MessageTracker(timer, env, replySender, false, std::move(bucketLock), - std::move(msg), ThrottleToken())); + {}, std::move(msg), ThrottleToken())); } void @@ -102,6 +144,7 @@ MessageTracker::sendReply() { if (hasReply()) { getReply().getTrace().addChild(_context.steal_trace()); if (_updateBucketInfo) { + assert(_bucketLock); if (getReply().getResult().success()) { _env.setBucketInfo(*this, _bucketLock->getBucket()); } @@ -163,7 +206,7 @@ MessageTracker::generateReply(api::StorageCommand& cmd) std::shared_ptr<FileStorHandler::OperationSyncPhaseDoneNotifier> MessageTracker::sync_phase_done_notifier_or_nullptr() const { - if (_bucketLock->wants_sync_phase_done_notification()) { + if (_bucketLock && _bucketLock->wants_sync_phase_done_notification()) { return _bucketLock; } return {}; @@ -236,7 +279,7 @@ PersistenceUtil::setBucketInfo(MessageTracker& tracker, const document::Bucket & { api::BucketInfo info = getBucketInfo(bucket); - static_cast<api::BucketInfoReply&>(tracker.getReply()).setBucketInfo(info); + dynamic_cast<api::BucketInfoReply&>(tracker.getReply()).setBucketInfo(info); updateBucketDatabase(bucket, info); } diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 900f301252e..67e96befe00 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -25,12 +25,52 @@ namespace storage { class PersistenceUtil; +struct DeferredReplySenderStub : MessageSender { + std::mutex _mutex; + std::vector<std::shared_ptr<api::StorageReply>> _deferred_replies; + + DeferredReplySenderStub(); + ~DeferredReplySenderStub() override; + + void sendCommand(const std::shared_ptr<api::StorageCommand>&) override { + abort(); // Not supported + } + void sendReply(const std::shared_ptr<api::StorageReply>& reply) override { + std::lock_guard lock(_mutex); + _deferred_replies.emplace_back(reply); + } +}; + +class AsyncMessageBatch { + std::shared_ptr<FileStorHandler::BucketLockInterface> _bucket_lock; + const PersistenceUtil& _env; + MessageSender& _reply_sender; + DeferredReplySenderStub _deferred_sender_stub; +public: + AsyncMessageBatch(std::shared_ptr<FileStorHandler::BucketLockInterface> bucket_lock, + const PersistenceUtil& env, + MessageSender& reply_sender) noexcept; + // Triggered by last referencing batched MessageTracker being destroyed. + // Fetches bucket info, updates DB and sends all deferred replies with the new bucket info. + ~AsyncMessageBatch(); + + [[nodiscard]] MessageSender& deferred_sender_stub() noexcept { return _deferred_sender_stub; } +}; + class MessageTracker { public: using UP = std::unique_ptr<MessageTracker>; - MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, - FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, + MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & reply_sender, + FileStorHandler::BucketLockInterface::SP bucket_lock, std::shared_ptr<api::StorageMessage> msg, + ThrottleToken throttle_token); + + // For use with batching where bucket lock is held separately and bucket info + // is _not_ fetched or updated per message. + MessageTracker(const framework::MilliSecTimer& timer, const PersistenceUtil& env, + std::shared_ptr<AsyncMessageBatch> batch, + MessageSender& deferred_reply_sender, + std::shared_ptr<api::StorageMessage> msg, ThrottleToken throttle_token); ~MessageTracker(); @@ -58,48 +98,53 @@ public: * commands like merge. */ void dontReply() { _sendReply = false; } - bool hasReply() const { return bool(_reply); } - const api::StorageReply & getReply() const { + [[nodiscard]] bool hasReply() const { return bool(_reply); } + [[nodiscard]] const api::StorageReply & getReply() const { return *_reply; } - api::StorageReply & getReply() { + [[nodiscard]] api::StorageReply & getReply() { return *_reply; } - std::shared_ptr<api::StorageReply> && stealReplySP() && { + [[nodiscard]] std::shared_ptr<api::StorageReply> && stealReplySP() && { return std::move(_reply); } void generateReply(api::StorageCommand& cmd); - api::ReturnCode getResult() const { return _result; } + [[nodiscard]] api::ReturnCode getResult() const { return _result; } - spi::Context & context() { return _context; } - document::BucketId getBucketId() const { + [[nodiscard]] spi::Context & context() { return _context; } + [[nodiscard]] document::BucketId getBucketId() const { return _bucketLock->getBucket().getBucketId(); } void sendReply(); - bool checkForError(const spi::Result& response); + [[nodiscard]] bool checkForError(const spi::Result& response); // Returns a non-nullptr notifier instance iff the underlying operation wants to be notified - // when the sync phase is complete. Otherwise returns a nullptr shared_ptr. - std::shared_ptr<FileStorHandler::OperationSyncPhaseDoneNotifier> sync_phase_done_notifier_or_nullptr() const; + // when the sync phase is complete. Otherwise, returns a nullptr shared_ptr. + [[nodiscard]] std::shared_ptr<FileStorHandler::OperationSyncPhaseDoneNotifier> sync_phase_done_notifier_or_nullptr() const; static MessageTracker::UP createForTesting(const framework::MilliSecTimer & timer, PersistenceUtil & env, MessageSender & replySender, FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); private: - MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, - FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, + MessageTracker(const framework::MilliSecTimer& timer, const PersistenceUtil& env, + MessageSender& reply_sender, bool update_bucket_info, + std::shared_ptr<FileStorHandler::BucketLockInterface> bucket_lock, + std::shared_ptr<AsyncMessageBatch> part_of_batch, + std::shared_ptr<api::StorageMessage> msg, ThrottleToken throttle_token); [[nodiscard]] bool count_result_as_failure() const noexcept; bool _sendReply; bool _updateBucketInfo; + // Either _bucketLock or _part_of_batch must be set, never both at the same time FileStorHandler::BucketLockInterface::SP _bucketLock; + std::shared_ptr<AsyncMessageBatch> _part_of_batch; // nullptr if not batched std::shared_ptr<api::StorageMessage> _msg; ThrottleToken _throttle_token; spi::Context _context; @@ -117,8 +162,6 @@ public: struct LockResult { std::shared_ptr<FileStorHandler::BucketLockInterface> lock; LockResult() : lock() {} - - bool bucketExisted() const { return bool(lock); } }; PersistenceUtil(const ServiceLayerComponent&, FileStorHandler& fileStorHandler, diff --git a/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h index 277e7b4fdfd..93b305370e0 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h @@ -13,7 +13,7 @@ namespace storage::framework::defaultimplementation { -struct RealClock : public Clock { +struct RealClock final : public Clock { vespalib::steady_time getMonotonicTime() const override; vespalib::system_time getSystemTime() const override; }; diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt index 67377c6cba3..25bf1ced552 100644 --- a/storageserver/src/apps/storaged/CMakeLists.txt +++ b/storageserver/src/apps/storaged/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_executable(storageserver_storaged_app DEPENDS storageserver_storageapp protobuf::libprotobuf + absl::failure_signal_handler ) vespa_add_target_package_dependency(storageserver_storaged_app Protobuf) diff --git a/storageserver/src/apps/storaged/storage.cpp b/storageserver/src/apps/storaged/storage.cpp index fe3bf696e9a..cffc03a585b 100644 --- a/storageserver/src/apps/storaged/storage.cpp +++ b/storageserver/src/apps/storaged/storage.cpp @@ -21,6 +21,7 @@ #include <vespa/config/helper/configgetter.hpp> #include <vespa/vespalib/util/signalhandler.h> #include <google/protobuf/message_lite.h> +#include <absl/debugging/failure_signal_handler.h> #include <iostream> #include <csignal> #include <cstdlib> @@ -213,8 +214,15 @@ int StorageApp::main(int argc, char **argv) } // storage int main(int argc, char **argv) { + absl::FailureSignalHandlerOptions opts; + // See `searchcore/src/apps/proton/proton.cpp` for parameter and handler ordering rationale. + opts.call_previous_handler = true; + opts.use_alternate_stack = false; + absl::InstallFailureSignalHandler(opts); + vespalib::SignalHandler::PIPE.ignore(); vespalib::SignalHandler::enable_cross_thread_stack_tracing(); + storage::StorageApp app; storage::sigtramp = &app; int retval = app.main(argc,argv); diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp index abeaf06b4ad..d1eed61ba48 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp @@ -88,7 +88,17 @@ ServiceLayerProcess::updateConfig() bool ServiceLayerProcess::configUpdated() { - return Process::configUpdated(); + bool changed = Process::configUpdated(); + if (_persistence_cfg_handle->isChanged()) { + changed = true; + } + if (_visitor_cfg_handle->isChanged()) { + changed = true; + } + if (_filestor_cfg_handle->isChanged()) { + changed = true; + } + return changed; } void diff --git a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp index 5c1e15e0c5c..72e024d8f3b 100644 --- a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp +++ b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp @@ -13,6 +13,8 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vsm/searcher/mock_field_searcher_env.h> #include <vespa/vsm/searcher/nearest_neighbor_field_searcher.h> +#include <vespa/vespalib/objects/nbostream.h> + using namespace search::attribute::test; using namespace search::attribute; diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp index ce9636895b4..84c3a542661 100644 --- a/streamingvisitors/src/tests/searcher/searcher_test.cpp +++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp @@ -75,31 +75,47 @@ std::string_view maybe_consume_into(std::string_view str, T& val_out) { return str.substr(ptr - str.data()); } -// Parse optional max edits and prefix lock length from term string. +// Parse optional prefix match mode, max edits and prefix lock length from term string. // Syntax: -// "term" -> {2, 0, "term"} (default max edits & prefix length) -// "{1}term" -> {1, 0, "term"} -// "{1,3}term" -> {1, 3, "term"} +// "term" -> {2, 0, false, "term"} (default max edits, prefix length and prefix match mode) +// "{p}term" -> {2, 0, true, "term"} +// "{1}term" -> {1, 0, false, "term"} +// "{p1}term" -> {1, 0, true, "term"} +// "{1,3}term" -> {1, 3, false, "term"} +// "{p1,3}term" -> {1, 3, true, "term"} +// .. and so on // // Note: this is not a "proper" parser (it accepts empty numeric values); only for testing! -std::tuple<uint8_t, uint32_t, std::string_view> parse_fuzzy_params(std::string_view term) { +std::tuple<uint8_t, uint32_t, bool, std::string_view> parse_fuzzy_params(std::string_view term) { + std::string_view orig_term = term; if (term.empty() || term[0] != '{') { - return {2, 0, term}; + return {2, 0, false, term}; } + term = term.substr(1); // skip '{' uint8_t max_edits = 2; uint32_t prefix_length = 0; - term = maybe_consume_into(term.substr(1), max_edits); + bool prefix_match = false; + if (!term.empty() && term[0] == 'p') { + prefix_match = true; + term = term.substr(1); // skip 'p' + } + if (!term.empty() && term[0] == '}') { + return {2, 0, prefix_match, term.substr(1)}; + } + term = maybe_consume_into(term, max_edits); if (term.empty() || (term[0] != ',' && term[0] != '}')) { - throw std::invalid_argument("malformed fuzzy params at (or after) max_edits"); + throw std::invalid_argument("malformed fuzzy params at (or after) max_edits: " + + std::string(term) + " in string " + std::string(orig_term)); } if (term[0] == '}') { - return {max_edits, prefix_length, term.substr(1)}; + return {max_edits, prefix_length, prefix_match, term.substr(1)}; } term = maybe_consume_into(term.substr(1), prefix_length); if (term.empty() || term[0] != '}') { - throw std::invalid_argument("malformed fuzzy params at (or after) prefix_length"); + throw std::invalid_argument("malformed fuzzy params at (or after) prefix_length: " + + std::string(term) + " in string " + std::string(orig_term)); } - return {max_edits, prefix_length, term.substr(1)}; + return {max_edits, prefix_length, prefix_match, term.substr(1)}; } } @@ -115,9 +131,10 @@ private: if (pt.second == TermType::REGEXP) { qtv.push_back(std::make_unique<RegexpTerm>(eqnr.create(), pt.first, effective_index, TermType::REGEXP, normalizing)); } else if (pt.second == TermType::FUZZYTERM) { - auto [max_edits, prefix_length, actual_term] = parse_fuzzy_params(pt.first); + auto [max_edits, prefix_lock_length, prefix_match, actual_term] = parse_fuzzy_params(pt.first); qtv.push_back(std::make_unique<FuzzyTerm>(eqnr.create(), vespalib::stringref(actual_term.data(), actual_term.size()), - effective_index, TermType::FUZZYTERM, normalizing, max_edits, prefix_length)); + effective_index, TermType::FUZZYTERM, normalizing, max_edits, + prefix_lock_length, prefix_match)); } else { qtv.push_back(std::make_unique<QueryTerm>(eqnr.create(), pt.first, effective_index, pt.second, normalizing)); } @@ -531,25 +548,29 @@ testStrChrFieldSearcher(StrChrFieldSearcher & fs) return true; } -TEST("parsing of test-only fuzzy term params can extract numeric values") { +void check_fuzzy_param_parsing(std::string_view term, std::string_view exp_term, + uint8_t exp_max_edits, uint32_t exp_prefix_length, bool exp_prefix) +{ uint8_t max_edits = 0; - uint32_t prefix_length = 1234; + uint32_t prefix_length = 0; + bool prefix = false; std::string_view out; - std::tie(max_edits, prefix_length, out) = parse_fuzzy_params("myterm"); - EXPECT_EQUAL(max_edits, 2u); - EXPECT_EQUAL(prefix_length, 0u); - EXPECT_EQUAL(out, "myterm"); + std::tie(max_edits, prefix_length, prefix, out) = parse_fuzzy_params(term); + EXPECT_EQUAL(static_cast<uint32_t>(max_edits), static_cast<uint32_t>(exp_max_edits)); // don't print as char... + EXPECT_EQUAL(prefix_length, exp_prefix_length); + EXPECT_EQUAL(prefix, exp_prefix); + EXPECT_EQUAL(out, exp_term); - std::tie(max_edits, prefix_length, out) = parse_fuzzy_params("{3}myterm"); - EXPECT_EQUAL(max_edits, 3u); - EXPECT_EQUAL(prefix_length, 0u); - EXPECT_EQUAL(out, "myterm"); +} - std::tie(max_edits, prefix_length, out) = parse_fuzzy_params("{2,70}myterm"); - EXPECT_EQUAL(max_edits, 2u); - EXPECT_EQUAL(prefix_length, 70u); - EXPECT_EQUAL(out, "myterm"); +TEST("parsing of test-only fuzzy term params can extract expected values") { + check_fuzzy_param_parsing("myterm", "myterm", 2, 0, false); + check_fuzzy_param_parsing("{3}myterm", "myterm", 3, 0, false); + check_fuzzy_param_parsing("{p}myterm", "myterm", 2, 0, true); + check_fuzzy_param_parsing("{p1}myterm", "myterm", 1, 0, true); + check_fuzzy_param_parsing("{2,70}myterm", "myterm", 2, 70, false); + check_fuzzy_param_parsing("{p2,70}myterm", "myterm", 2, 70, true); } TEST("verify correct term parsing") { @@ -640,6 +661,14 @@ TEST("utf8 substring search with empty term") assertFieldInfo(fs, "", "abc", QTFieldInfo().setFieldLength(0)); } +Hits is_hit() { + return Hits().add({0, 0}); +} + +Hits no_hits() { + return {}; +} + TEST("utf8 suffix search") { UTF8SuffixStringFieldSearcher fs(0); std::string field = "operators and operator overloading"; @@ -837,6 +866,37 @@ TEST("utf8 flexible searcher fuzzy matching treats field as 1 word") { TEST_DO(assertFieldInfo(fs, "%{1}foo", "foo bar baz", QTFieldInfo(0, 0, 1))); } +TEST("utf8 flexible searcher supports fuzzy prefix matching") { + UTF8FlexibleStringFieldSearcher fs(0); + TEST_DO(assertString(fs, "%{p0}z", "zoid", is_hit())); + TEST_DO(assertString(fs, "%{p0}zo", "zoid", is_hit())); + TEST_DO(assertString(fs, "%{p0}zo", "Zoid", is_hit())); // uncased + TEST_DO(assertString(fs, "%{p0}Zo", "zoid", is_hit())); // uncased + TEST_DO(assertString(fs, "%{p0}zoid", "zoid", is_hit())); + TEST_DO(assertString(fs, "%{p0}x", "zoid", no_hits())); + TEST_DO(assertString(fs, "%{p1}zo", "boid", is_hit())); + TEST_DO(assertString(fs, "%{p1}zo", "blid", no_hits())); + TEST_DO(assertString(fs, "%{p1}yam", "hamburger", is_hit())); + TEST_DO(assertString(fs, "%{p1}yam", "humbug", no_hits())); + TEST_DO(assertString(fs, "%{p2}yam", "humbug", is_hit())); + TEST_DO(assertString(fs, "%{p2}catfo", "dogfood", no_hits())); + TEST_DO(assertString(fs, "%{p3}catfo", "dogfood", is_hit())); + TEST_DO(assertString(fs, "%{p100}abcd", "anything you want", is_hit())); // trivially matches +} + +TEST("utf8 flexible searcher supports fuzzy prefix matching combined with prefix locking") { + UTF8FlexibleStringFieldSearcher fs(0); + TEST_DO(assertString(fs, "%{p0,4}zoid", "zoid", is_hit())); + TEST_DO(assertString(fs, "%{p0,4}zoidber", "zoidberg", is_hit())); + TEST_DO(assertString(fs, "%{p1,4}zoidber", "zoidburg", is_hit())); + TEST_DO(assertString(fs, "%{p1,4}zoidber", "zoidblurgh", no_hits())); + TEST_DO(assertString(fs, "%{p1,4}zoidbe", "zoidblurgh", is_hit())); + TEST_DO(assertString(fs, "%{p1,4}zoidberg", "boidberg", no_hits())); + TEST_DO(assertString(fs, "%{p1,4}zoidber", "zoidburger", is_hit())); + TEST_DO(assertString(fs, "%{p1,4}zoidber", "zoidbananas", no_hits())); + TEST_DO(assertString(fs, "%{p2,4}zoidber", "zoidbananas", is_hit())); +} + TEST("bool search") { BoolFieldSearcher fs(0); TEST_DO(assertBool(fs, "true", true, true)); diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp index 816317bf86d..2fd23100f46 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp @@ -131,7 +131,7 @@ NearestNeighborFieldSearcher::onValue(const document::FieldValue& fv) _attr->add(*tfv->getAsTensorPtr(), 1); for (auto& elem : _calcs) { double distance_limit = elem->heap.distanceLimit(); - double distance = elem->calc->calc_with_limit(scratch_docid, distance_limit); + double distance = elem->calc->calc_with_limit<false>(scratch_docid, distance_limit); if (distance <= distance_limit) { elem->node->set_distance(distance); } diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java index dd76827dd2b..1ae5171c93a 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -5,7 +5,6 @@ import java.net.URI; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.Map; /** @@ -34,7 +33,7 @@ public interface Endpoint { /** Creates a request against the endpoint, with the given path. */ default HttpRequest.Builder request(String path) { - return request(path, Collections.emptyMap()); + return request(path, Map.of()); } } diff --git a/testutil/src/main/java/com/yahoo/test/OrderTester.java b/testutil/src/main/java/com/yahoo/test/OrderTester.java index 391eb5c4ebd..6f83cd723b8 100644 --- a/testutil/src/main/java/com/yahoo/test/OrderTester.java +++ b/testutil/src/main/java/com/yahoo/test/OrderTester.java @@ -2,7 +2,6 @@ package com.yahoo.test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** @@ -25,7 +24,7 @@ public abstract class OrderTester<T extends Comparable<? super T>> { @SafeVarargs @SuppressWarnings("varargs") private OrderTester<T> addGroup(T... group) { - groups.add(Arrays.asList(group)); + groups.add(List.of(group)); return this; } diff --git a/testutil/src/test/java/com/yahoo/test/MatchersTestCase.java b/testutil/src/test/java/com/yahoo/test/MatchersTestCase.java index 839856db7f2..c36818e4f7f 100644 --- a/testutil/src/test/java/com/yahoo/test/MatchersTestCase.java +++ b/testutil/src/test/java/com/yahoo/test/MatchersTestCase.java @@ -1,13 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.test; -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.hamcrest.Matcher; import org.junit.Test; +import java.util.List; + /** * Tests for com.yahoo.test.Matchers. * @@ -18,16 +19,9 @@ public class MatchersTestCase { @Test public final void testHasItemWithMethodObjectString() { @SuppressWarnings("rawtypes") - final Matcher<Iterable> m = Matchers.hasItemWithMethod("nalle", - "toLowerCase"); - assertEquals( - false, - m.matches(Arrays.asList(new Object[] { Integer.valueOf(1), - Character.valueOf('c'), "blbl" }))); - assertEquals( - true, - m.matches(Arrays.asList(new Object[] { Character.valueOf('c'), - "NALLE" }))); + final Matcher<Iterable> m = Matchers.hasItemWithMethod("nalle", "toLowerCase"); + assertFalse(m.matches(List.of(new Object[]{1, 'c', "blbl"}))); + assertTrue(m.matches(List.of(new Object[]{'c', "NALLE"}))); } } diff --git a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java index 415bed7419d..279bffcd49e 100644 --- a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java +++ b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java @@ -21,7 +21,6 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; -import java.util.Collections; import java.util.List; /** @@ -173,7 +172,7 @@ public class ApplicationMojo extends AbstractMojo { } private static <T> List<T> emptyListIfNull(List<T> modules) { - return modules == null ? Collections.emptyList(): modules; + return modules == null ? List.of(): modules; } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPrincipal.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPrincipal.java index 08d289e216b..2a1113be26a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPrincipal.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPrincipal.java @@ -6,8 +6,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import static java.util.Collections.emptyList; - /** * @author bjorncs */ @@ -18,11 +16,11 @@ public class AthenzPrincipal implements Principal { private final List<AthenzRole> roles; public AthenzPrincipal(AthenzIdentity athenzIdentity) { - this(athenzIdentity, null, emptyList()); + this(athenzIdentity, null, List.of()); } public AthenzPrincipal(AthenzIdentity athenzIdentity, NToken nToken) { - this(athenzIdentity, nToken, emptyList()); + this(athenzIdentity, nToken, List.of()); } public AthenzPrincipal(AthenzIdentity identity, List<AthenzRole> roles) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index c4c8fac87b4..f054abf2bd1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -43,7 +43,6 @@ import javax.net.ssl.SSLContext; import java.net.URI; import java.security.PublicKey; import java.time.Instant; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -88,7 +87,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { HttpUriRequest request = RequestBuilder.put() .setUri(uri) .addHeader(createCookieHeader(oAuthCredentials)) - .setEntity(toJsonStringEntity(new TenancyRequestEntity(tenantDomain, providerService, Collections.emptyList()))) + .setEntity(toJsonStringEntity(new TenancyRequestEntity(tenantDomain, providerService, List.of()))) .build(); execute(request, response -> readEntity(response, Void.class)); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index ed3a52abb87..2c8908a89a6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -17,15 +17,11 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.cert.Certificate; + import java.security.cert.X509Certificate; import java.util.List; -import java.util.Spliterator; -import java.util.Spliterators; + import java.util.stream.Stream; -import java.util.stream.StreamSupport; /** * A {@link ServiceIdentityProvider} that provides the credentials stored on file system. diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 40a23ab5904..63c966004e5 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -123,9 +123,9 @@ class AthenzCredentialsService { private Optional<AthenzCredentials> tryReadCredentialsFromDisk() { Optional<PrivateKey> privateKey = SiaUtils.readPrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity); - if (!privateKey.isPresent()) return Optional.empty(); + if (privateKey.isEmpty()) return Optional.empty(); Optional<X509Certificate> certificate = SiaUtils.readCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity); - if (!certificate.isPresent()) return Optional.empty(); + if (certificate.isEmpty()) return Optional.empty(); if (isExpired(certificate.get())) { return Optional.empty(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java index 5569eef192c..34324ef18e6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java @@ -38,7 +38,6 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -277,7 +276,7 @@ public final class LegacyAthenzIdentityProviderImpl extends AbstractComponent im @Override public List<X509Certificate> getIdentityCertificate() { - return Collections.singletonList(credentials.getCertificate()); + return List.of(credentials.getCertificate()); } @Override diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index 46d8976b1ce..2532a394f4e 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -18,8 +18,8 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.security.KeyPair; import java.time.Instant; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION; @@ -41,7 +41,7 @@ public class IdentityDocumentSignerTest { private static final String configserverHostname = "configserverhostname"; private static final String instanceHostname = "instancehostname"; private static final Instant createdAt = Instant.EPOCH; - private static final HashSet<String> ipAddresses = new HashSet<>(Arrays.asList("1.2.3.4", "::1")); + private static final HashSet<String> ipAddresses = new HashSet<>(List.of("1.2.3.4", "::1")); private static final ClusterType clusterType = ClusterType.CONTAINER; private static final URI ztsUrl = URI.create("https://foo"); private static final AthenzIdentity serviceIdentity = new AthenzService("vespa", "node"); diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceCsrGeneratorTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceCsrGeneratorTest.java index 1f9ad2ced64..322e9ffaf82 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceCsrGeneratorTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceCsrGeneratorTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test; import javax.security.auth.x500.X500Principal; import java.security.KeyPair; -import java.util.Collections; import java.util.Set; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; @@ -36,7 +35,7 @@ public class InstanceCsrGeneratorTest { VespaUniqueInstanceId vespaUniqueInstanceId = VespaUniqueInstanceId.fromDottedString("0.default.default.foo-app.vespa.us-north-1.prod.node"); KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr(service, vespaUniqueInstanceId, Collections.emptySet(), ClusterType.CONTAINER, keyPair); + Pkcs10Csr csr = csrGenerator.generateInstanceCsr(service, vespaUniqueInstanceId, Set.of(), ClusterType.CONTAINER, keyPair); assertEquals(new X500Principal(String.format("OU=%s, CN=%s", PROVIDER_SERVICE, ATHENZ_SERVICE)), csr.getSubject()); var actualSans = Set.copyOf(csr.getSubjectAlternativeNames()); var expectedSans = Set.of( diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 5cd1d3bf9b6..027682e1052 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -598,7 +598,7 @@ public class DocumentGenMojo extends AbstractMojo { private static void exportFieldSetDefinition(Set<FieldSet> fieldSets, Writer out, int ind) throws IOException { out.write(ind(ind) + "java.util.Map<java.lang.String, java.util.Collection<java.lang.String>> fieldSets = new java.util.HashMap<>();\n"); for (FieldSet fieldSet : fieldSets) { - out.write(ind(ind) + "fieldSets.put(\"" + fieldSet.getName() + "\", java.util.Arrays.asList("); + out.write(ind(ind) + "fieldSets.put(\"" + fieldSet.getName() + "\", java.util.List.of("); int count = 0; for (String field : fieldSet.getFieldNames()) { out.write("\"" + field + "\""); diff --git a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index 7101b8452ed..6fd17154aa8 100644 --- a/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client-api/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -9,7 +9,6 @@ import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -27,7 +26,7 @@ public interface FeedClientBuilder { * Creates a builder for a single feed container endpoint. * This is for feeding against a container cluster with a load balancer in front of it. **/ - static FeedClientBuilder create(URI endpoint) { return create(Collections.singletonList(endpoint)); } + static FeedClientBuilder create(URI endpoint) { return create(List.of(endpoint)); } /** * Creates a builder which <em>distributes</em> the feed across the given feed container endpoints. diff --git a/vespa-feed-client-api/src/test/java/ai/vespa/feed/client/FeedClientTest.java b/vespa-feed-client-api/src/test/java/ai/vespa/feed/client/FeedClientTest.java index 4b8e91a35a6..d64df85aea3 100644 --- a/vespa-feed-client-api/src/test/java/ai/vespa/feed/client/FeedClientTest.java +++ b/vespa-feed-client-api/src/test/java/ai/vespa/feed/client/FeedClientTest.java @@ -5,7 +5,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -89,9 +88,9 @@ class FeedClientTest { CompletableFuture<Result> f3 = CompletableFuture.completedFuture(new MyResult()); MultiFeedException multiException = assertThrows(MultiFeedException.class, () -> FeedClient.await(f1, f2, f3)); - Set<DocumentId> expectedDocsIds = new HashSet<>(Arrays.asList(docId1, docId2)); + Set<DocumentId> expectedDocsIds = new HashSet<>(List.of(docId1, docId2)); assertEquals(expectedDocsIds, new HashSet<>(multiException.documentIds())); - Set<FeedException> expectedExceptions = new HashSet<>(Arrays.asList(exceptionDoc1, exceptionDoc2)); + Set<FeedException> expectedExceptions = new HashSet<>(List.of(exceptionDoc1, exceptionDoc2)); assertEquals(expectedExceptions, new HashSet<>(multiException.feedExceptions())); assertEquals("2 feed operations failed", multiException.getMessage()); } diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index e07108a84ad..35faa89388b 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -18,7 +18,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; import java.time.Duration; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -26,7 +25,6 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import static ai.vespa.feed.client.FeedClientBuilder.Compression.auto; -import static ai.vespa.feed.client.FeedClientBuilder.Compression.none; /** * Parses command line arguments @@ -139,7 +137,7 @@ class CliArguments { Map<String, String> headers() throws CliArgumentsException { String[] rawArguments = arguments.getOptionValues(HEADER_OPTION); - if (rawArguments == null) return Collections.emptyMap(); + if (rawArguments == null) return Map.of(); Map<String, String> headers = new HashMap<>(); for (String rawArgument : rawArguments) { if (rawArgument.startsWith("\"") || rawArgument.startsWith("'")) { @@ -152,7 +150,7 @@ class CliArguments { if (colonIndex == -1) throw new CliArgumentsException("Invalid header: '" + rawArgument + "'"); headers.put(rawArgument.substring(0, colonIndex), rawArgument.substring(colonIndex + 1).trim()); } - return Collections.unmodifiableMap(headers); + return Map.copyOf(headers); } boolean sslHostnameVerificationDisabled() { return has(DISABLE_SSL_HOSTNAME_VERIFICATION_OPTION); } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 0268f1a4394..c271ac356e9 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -15,7 +15,6 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -195,7 +194,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { /** Sets client SSL certificate/key */ @Override public FeedClientBuilderImpl setCertificate(X509Certificate certificate, PrivateKey privateKey) { - return setCertificate(Collections.singletonList(certificate), privateKey); + return setCertificate(List.of(certificate), privateKey); } @Override diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java index 5454249d52e..df010a167f6 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java @@ -41,7 +41,6 @@ import java.net.Inet4Address; import java.net.InetSocketAddress; import java.net.URI; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -194,7 +193,7 @@ class JettyCluster implements Cluster { proxySslCtxFactory.setSslContext(b.constructProxySslContext()); try { proxySslCtxFactory.start(); } catch (Exception e) { throw new IOException(e); } httpClient.getProxyConfiguration().addProxy( - new HttpProxy(address, proxySslCtxFactory, new Origin.Protocol(Collections.singletonList("h2"), false))); + new HttpProxy(address, proxySslCtxFactory, new Origin.Protocol(List.of("h2"), false))); URI proxyUri = URI.create(endpointUri(b.proxy)); httpClient.getAuthenticationStore().addAuthenticationResult(new Authentication.Result() { @Override public URI getURI() { return proxyUri; } @@ -205,7 +204,7 @@ class JettyCluster implements Cluster { } else { // Assume insecure proxy uses HTTP/1.1 httpClient.getProxyConfiguration().addProxy( - new HttpProxy(address, false, new Origin.Protocol(Collections.singletonList("http/1.1"), false))); + new HttpProxy(address, false, new Origin.Protocol(List.of("http/1.1"), false))); // Bug in Jetty cause authentication result to be ignored for HTTP/1.1 CONNECT requests httpClient.getRequestListeners().add(new Request.Listener() { @Override diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/SslContextBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/SslContextBuilder.java index 36f708e6535..1b0ad7a6b81 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/SslContextBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/SslContextBuilder.java @@ -28,7 +28,6 @@ import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.security.spec.PKCS8EncodedKeySpec; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -81,7 +80,7 @@ class SslContextBuilder { keystore.setKeyEntry("cert", privateKey, new char[0], certificate.toArray(new Certificate[0])); } if (hasCaCertificateFile()) { - addCaCertificates(keystore, Arrays.asList(certificates(caCertificatesFile))); + addCaCertificates(keystore, List.of(certificates(caCertificatesFile))); } else if (hasCaCertificateInstance()) { addCaCertificates(keystore, caCertificates); } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java index c79bb7b4606..28bde16f457 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpFeedClientTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.time.Duration; -import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -42,7 +42,7 @@ class HttpFeedClientTest { @Override public void await() { throw new UnsupportedOperationException(); } @Override public CompletableFuture<HttpResponse> enqueue(DocumentId documentId, HttpRequest request) { return dispatch.get().apply(documentId, request); } } - FeedClient client = new HttpFeedClient(new FeedClientBuilderImpl(Collections.singletonList(URI.create("https://dummy:123"))).setDryrun(true), + FeedClient client = new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))).setDryrun(true), new DryrunCluster(), new MockRequestStrategy()); @@ -214,7 +214,7 @@ class HttpFeedClientTest { void testHandshake() { // dummy:123 does not exist, and results in a host-not-found exception. FeedException exception = assertThrows(FeedException.class, - () -> new HttpFeedClient(new FeedClientBuilderImpl(Collections.singletonList(URI.create("https://dummy:123"))))); + () -> new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))))); String message = exception.getMessage(); assertTrue(message.startsWith("failed handshake with server after "), message); assertTrue(message.contains("java.net.UnknownHostException"), message); @@ -237,19 +237,19 @@ class HttpFeedClientTest { // Old server, and speed-test. assertEquals("server does not support speed test; upgrade to a newer version", assertThrows(FeedException.class, - () -> new HttpFeedClient(new FeedClientBuilderImpl(Collections.singletonList(URI.create("https://dummy:123"))).setSpeedTest(true), + () -> new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))).setSpeedTest(true), cluster, null)) .getMessage()); // Old server. - new HttpFeedClient(new FeedClientBuilderImpl(Collections.singletonList(URI.create("https://dummy:123"))), + new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))), cluster, null); // New server. response.set(okResponse); - new HttpFeedClient(new FeedClientBuilderImpl(Collections.singletonList(URI.create("https://dummy:123"))), + new HttpFeedClient(new FeedClientBuilderImpl(List.of(URI.create("https://dummy:123"))), cluster, null); } diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java index 56eb741e302..f3aa10dd1fa 100644 --- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java +++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/PomXmlGeneratorTest.java @@ -6,7 +6,6 @@ import org.junit.Test; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -18,7 +17,7 @@ public class PomXmlGeneratorTest { @Test public void write_system_tests_pom_xml() throws IOException { - List<Path> artifacts = Arrays.asList( + List<Path> artifacts = List.of( Paths.get("components/my-comp.jar"), Paths.get("main.jar")); diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in index a296acf7039..0e8050d41b5 100644 --- a/vespabase/src/vespa-configserver.service.in +++ b/vespabase/src/vespa-configserver.service.in @@ -12,7 +12,7 @@ ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver LimitNOFILE=32768:262144 LimitCORE=0:infinity LimitNPROC=32768:409600 -LimitStack=8388608:16777216 +LimitSTACK=8388608:16777216 [Install] WantedBy=multi-user.target diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in index 272769b111f..bde505947f2 100644 --- a/vespabase/src/vespa.service.in +++ b/vespabase/src/vespa.service.in @@ -12,7 +12,7 @@ ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services LimitNOFILE=32768:262144 LimitCORE=0:infinity LimitNPROC=32768:409600 -LimitStack=8388608:16777216 +LimitSTACK=8388608:16777216 [Install] WantedBy=multi-user.target diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 594c5c8f398..b483d6977d6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -187,12 +187,17 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private final Metric metric; private final DocumentApiMetrics metrics; private final DocumentOperationParser parser; + private final long maxThrottled; + private final long maxThrottledAgeNS; private final DocumentAccess access; private final AsyncSession asyncSession; private final Map<String, StorageCluster> clusters; + private final Deque<Operation> operations; private final Deque<BooleanSupplier> visitOperations = new ConcurrentLinkedDeque<>(); + private final AtomicLong enqueued = new AtomicLong(); private final AtomicLong outstanding = new AtomicLong(); private final Map<VisitorControlHandler, VisitorSession> visits = new ConcurrentHashMap<>(); + private final ScheduledExecutorService dispatcher = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("document-api-handler-")); private final ScheduledExecutorService visitDispatcher = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("document-api-handler-visit-")); private final Map<String, Map<Method, Handler>> handlers = defineApi(); @@ -216,12 +221,16 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { this.parser = new DocumentOperationParser(documentmanagerConfig); this.metric = metric; this.metrics = new DocumentApiMetrics(metricReceiver, "documentV1"); + this.maxThrottled = executorConfig.maxThrottled(); + this.maxThrottledAgeNS = (long) (executorConfig.maxThrottledAge() * 1_000_000_000.0); this.access = access; this.asyncSession = access.createAsyncSession(new AsyncParameters()); this.clusters = parseClusters(clusterListConfig, bucketSpacesConfig); + this.operations = new ConcurrentLinkedDeque<>(); long resendDelayMS = SystemTimer.adjustTimeoutByDetectedHz(Duration.ofMillis(executorConfig.resendDelayMillis())).toMillis(); // TODO: Here it would be better to have dedicated threads with different wait depending on blocked or empty. + this.dispatcher.scheduleWithFixedDelay(this::dispatchEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); this.visitDispatcher.scheduleWithFixedDelay(this::dispatchVisitEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); } @@ -279,19 +288,27 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { visits.values().forEach(VisitorSession::abort); visits.values().forEach(VisitorSession::destroy); - // Shut down visitor dispatcher, so only we empty the queue of outstanding operations, and can be sure it is empty. + // Shut down both dispatchers, so only we empty the queues of outstanding operations, and can be sure they're empty. + dispatcher.shutdown(); visitDispatcher.shutdown(); - while ( ! (visitOperations.isEmpty()) && clock.instant().isBefore(doom)) { + while ( ! (operations.isEmpty() && visitOperations.isEmpty()) && clock.instant().isBefore(doom)) { + dispatchEnqueued(); dispatchVisitEnqueued(); } + if ( ! operations.isEmpty()) + log.log(WARNING, "Failed to empty request queue before shutdown timeout — " + operations.size() + " requests left"); + if ( ! visitOperations.isEmpty()) - log.log(WARNING, "Failed to empty visitor operations queue before shutdown timeout — " + visitOperations.size() + " operations left"); + log.log(WARNING, "Failed to empty visitor operations queue before shutdown timeout — " + operations.size() + " operations left"); try { while (outstanding.get() > 0 && clock.instant().isBefore(doom)) Thread.sleep(Math.max(1, Duration.between(clock.instant(), doom).toMillis())); + if ( ! dispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), MILLISECONDS)) + dispatcher.shutdownNow(); + if ( ! visitDispatcher.awaitTermination(Duration.between(clock.instant(), doom).toMillis(), MILLISECONDS)) visitDispatcher.shutdownNow(); } @@ -535,6 +552,30 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } /** Dispatches enqueued requests until one is blocked. */ + void dispatchEnqueued() { + try { + while (dispatchFirst()); + } + catch (Exception e) { + log.log(WARNING, "Uncaught exception in /document/v1 dispatch thread", e); + } + } + + /** Attempts to dispatch the first enqueued operations, and returns whether this was successful. */ + private boolean dispatchFirst() { + Operation operation = operations.poll(); + if (operation == null) + return false; + + if (operation.dispatch()) { + enqueued.decrementAndGet(); + return true; + } + operations.push(operation); + return false; + } + + /** Dispatches enqueued requests until one is blocked. */ void dispatchVisitEnqueued() { try { while (dispatchFirstVisit()); @@ -557,16 +598,36 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { return false; } + private long qAgeNS(HttpRequest request) { + Operation oldest = operations.peek(); + return (oldest != null) + ? (request.relativeCreatedAtNanoTime() - oldest.request.relativeCreatedAtNanoTime()) + : 0; + } + /** * Enqueues the given request and operation, or responds with "overload" if the queue is full, * and then attempts to dispatch an enqueued operation from the head of the queue. */ private void enqueueAndDispatch(HttpRequest request, ResponseHandler handler, Supplier<BooleanSupplier> operationParser) { - Operation operation = new Operation(request, handler, operationParser); - if ( ! operation.dispatch()) { + long numQueued = enqueued.incrementAndGet(); + if (numQueued > maxThrottled) { + enqueued.decrementAndGet(); overload(request, "Rejecting execution due to overload: " - + (long)asyncSession.getCurrentWindowSize() + " requests already enqueued", handler); + + maxThrottled + " requests already enqueued", handler); + return; + } + if (numQueued > 1) { + long ageNS = qAgeNS(request); + if (ageNS > maxThrottledAgeNS) { + enqueued.decrementAndGet(); + overload(request, "Rejecting execution due to overload: " + + maxThrottledAgeNS / 1_000_000_000.0 + " seconds worth of work enqueued", handler); + return; + } } + operations.offer(new Operation(request, handler, operationParser)); + dispatchFirst(); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 92a9c4df27d..75f77409447 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -33,7 +33,7 @@ import java.util.logging.Logger; * avoid using a threadpool that has no effect with all the extra that comes with it. V2 has one instance per thread * on the client, while this is one instance for all threads. * - * @author dybis + * @author Haakon Dybdahl */ class ClientFeederV3 { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java index 8b3f9e45a79..e5f826b8913 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -11,7 +11,7 @@ import com.yahoo.vespaxmlparser.FeedOperation; /** * Keeps an operation with its message. * - * @author dybis + * @author Haakon Dybdahl */ class DocumentOperationMessageV3 { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index ccd0075a58a..4c98dba87cf 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -27,7 +27,7 @@ import java.util.logging.Logger; * One client has one ClientFeederV3 shared between all client threads. * Contains logic for shutting down cleanly as the server is upgraded. * - * @author dybis + * @author Haakon Dybdahl */ public class FeedHandlerV3 extends ThreadedHttpRequestHandler { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java index c501fbfc500..792115f2570 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -11,7 +11,7 @@ import java.io.InputStream; /** * Class for creating FeedReader based on dataFormat. - * @author dybis + * @author Haakon Dybdahl */ public class FeedReaderFactory { private static final int MARK_READLIMIT = 200; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index bd1c66a36e5..1dd640004d3 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -15,7 +15,7 @@ import java.util.zip.GZIPInputStream; /** * This code is based on v2 code, but restructured so stream reading code is in one dedicated class. * - * @author dybis + * @author Haakon Dybdahl */ public class StreamReaderV3 { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 2d0b2de100e..58cf34712aa 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -71,7 +71,13 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -184,31 +190,55 @@ public class DocumentV1ApiTest { } @Test - public void testOverLoad() { + public void testOverLoadBySize() { RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); // OVERLOAD is a 429 access.session.expect((id, parameters) -> new Result(Result.ResultType.TRANSIENT_ERROR, Result.toError(Result.ResultType.TRANSIENT_ERROR))); var response1 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); var response2 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); + var response3 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); + assertSameJson("{" + + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + + " \"message\": \"Rejecting execution due to overload: 2 requests already enqueued\"" + + "}", response3.readAll()); + assertEquals(429, response3.getStatus()); + + access.session.expect((id, parameters) -> new Result(Result.ResultType.FATAL_ERROR, Result.toError(Result.ResultType.FATAL_ERROR))); + handler.dispatchEnqueued(); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + - " \"message\": \"Rejecting execution due to overload: 20 requests already enqueued\"" + + " \"message\": \"[FATAL_ERROR @ localhost]: FATAL_ERROR\"" + "}", response1.readAll()); - assertEquals(429, response1.getStatus()); + assertEquals(500, response1.getStatus()); + assertSameJson("{" + + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + + " \"message\": \"[FATAL_ERROR @ localhost]: FATAL_ERROR\"" + + "}", response2.readAll()); + assertEquals(500, response2.getStatus()); + driver.close(); + } + @Test + public void testOverLoadByAge() { + RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); + // OVERLOAD is a 429 + access.session.expect((id, parameters) -> new Result(Result.ResultType.TRANSIENT_ERROR, Result.toError(Result.ResultType.TRANSIENT_ERROR))); + var response1 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); + try { Thread.sleep(3_000); } catch (InterruptedException e) {} + var response2 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + - " \"message\": \"Rejecting execution due to overload: 20 requests already enqueued\"" + + " \"message\": \"Rejecting execution due to overload: 1.0 seconds worth of work enqueued\"" + "}", response2.readAll()); - assertEquals(429, response1.getStatus()); + assertEquals(429, response2.getStatus()); access.session.expect((id, parameters) -> new Result(Result.ResultType.FATAL_ERROR, Result.toError(Result.ResultType.FATAL_ERROR))); - var response3 = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two", POST, "{\"fields\": {}}"); + handler.dispatchEnqueued(); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + " \"message\": \"[FATAL_ERROR @ localhost]: FATAL_ERROR\"" + - "}", response3.readAll()); - assertEquals(500, response3.getStatus()); + "}", response1.readAll()); + assertEquals(500, response1.getStatus()); driver.close(); } @@ -1006,6 +1036,78 @@ public class DocumentV1ApiTest { }); } + @Test + public void testThroughput() throws InterruptedException { + DocumentOperationExecutorConfig executorConfig = new DocumentOperationExecutorConfig.Builder().build(); + handler = new DocumentV1ApiHandler(clock, Duration.ofMillis(1), metric, metrics, access, docConfig, + executorConfig, clusterConfig, bucketConfig); + + int writers = 4; + int queueFill = executorConfig.maxThrottled() - writers; + RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); + ScheduledExecutorService writer = Executors.newScheduledThreadPool(writers); + ScheduledExecutorService reader = Executors.newScheduledThreadPool(1); + ScheduledExecutorService replier = Executors.newScheduledThreadPool(writers); + BlockingQueue<RequestHandlerTestDriver.MockResponseHandler> responses = new LinkedBlockingQueue<>(); + + Response success = new Response(0, null, Response.Outcome.SUCCESS); + int docs = 1 << 14; + assertTrue(docs >= writers); + AtomicReference<com.yahoo.jdisc.Response> failed = new AtomicReference<>(); + + CountDownLatch latch = new CountDownLatch(docs); + reader.execute(() -> { + while ( ! reader.isShutdown()) { + try { + var response = responses.take(); + response.awaitResponse().readAll(); + if (response.getStatus() != 200) + failed.set(response.getResponse()); + latch.countDown(); + } + catch (InterruptedException e) { break; } + } + }); + + // Fill the handler resend queue. + long startNanos = System.nanoTime(); + CountDownLatch setup = new CountDownLatch(queueFill); + access.session.expect((id, parameters) -> { + setup.countDown(); + return new Result(Result.ResultType.TRANSIENT_ERROR, Result.toError(Result.ResultType.TRANSIENT_ERROR)); + }); + for (int i = 0; i < queueFill; i++) { + int j = i; + writer.execute(() -> { + responses.add(driver.sendRequest("http://localhost/document/v1/ns/music/docid/" + j, + POST, + "{ \"fields\": { \"artist\": \"Sigrid\" } }")); + }); + } + setup.await(); + + // Let "messagebus" start accepting messages. + access.session.expect((id, parameters) -> { + replier.schedule(() -> parameters.responseHandler().get().handleResponse(success), 10, TimeUnit.MILLISECONDS); + return new Result(0); + }); + // Send the rest of the documents. Rely on resender to empty queue of throttled operations. + for (int i = queueFill; i < docs; i++) { + int j = i; + writer.execute(() -> { + responses.add(driver.sendRequest("http://localhost/document/v1/ns/music/docid/" + j, + POST, + "{ \"fields\": { \"artist\": \"Sigrid\" } }")); + }); + } + latch.await(); + System.err.println(docs + " requests in " + (System.nanoTime() - startNanos) * 1e-9 + " seconds"); + + assertNull(failed.get()); + driver.close(); + } + + static class MockDocumentAccess extends DocumentAccess { private final AtomicReference<Consumer<VisitorParameters>> expectations = new AtomicReference<>(); @@ -1121,7 +1223,7 @@ public class DocumentV1ApiTest { @Override public double getCurrentWindowSize() { - return 20; + throw new AssertionError("Not used"); } public void expect(BiFunction<Object, DocumentOperationParameters, Result> expectations) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java index 4636f96a4bb..52a58474238 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -7,7 +7,6 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; @@ -20,17 +19,17 @@ import static org.junit.Assert.assertTrue; */ public class VersionsTestCase { - private static final List<String> EMPTY = Collections.emptyList(); - private static final List<String> ONE_TWO = Arrays.asList("1", "2"); - private static final List<String> ONE_THREE = Arrays.asList("1", "3"); - private static final List<String> TWO_THREE = Arrays.asList("3", "2"); + private static final List<String> EMPTY = List.of(); + private static final List<String> ONE_TWO = List.of("1", "2"); + private static final List<String> ONE_THREE = List.of("1", "3"); + private static final List<String> TWO_THREE = List.of("3", "2"); private static final List<String> ONE_NULL_THREE = Arrays.asList("1", null, "3"); - private static final List<String> ONE_COMMA_THREE = Collections.singletonList("1, 3"); - private static final List<String> ONE_EMPTY_THREE = Arrays.asList("1", "", "3"); - private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000"); - private static final List<String> THREE_TOO_LARGE_NUMBER = Arrays.asList("3", "1000000000"); - private static final List<String> THREE_COMMA_TOO_LARGE_NUMBER = Arrays.asList("3,1000000000"); - private static final List<String> GARBAGE = Collections.singletonList("garbage"); + private static final List<String> ONE_COMMA_THREE = List.of("1, 3"); + private static final List<String> ONE_EMPTY_THREE = List.of("1", "", "3"); + private static final List<String> TOO_LARGE_NUMBER = List.of("1000000000"); + private static final List<String> THREE_TOO_LARGE_NUMBER = List.of("3", "1000000000"); + private static final List<String> THREE_COMMA_TOO_LARGE_NUMBER = List.of("3,1000000000"); + private static final List<String> GARBAGE = List.of("garbage"); @Test public void testEmpty() { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java index 9de368c24df..6d05af112ce 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -9,7 +9,7 @@ import java.io.InputStream; /** * For creating MockReader of innput stream. - * @author dybis + * @author Haakon Dybdahl */ public class MockFeedReaderFactory extends FeedReaderFactory { diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java index 8d8904f042f..9a7d8279b1b 100755..100644 --- a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -18,7 +18,6 @@ import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -174,7 +173,7 @@ public class DummyReceiver implements MessageHandler { LogSetup.initVespaLogging("dummyreceiver"); DummyReceiver rcv = new DummyReceiver(); - if (!rcv.parseArgs(new LinkedList<>(Arrays.asList(args))) && !rcv.helpOption) { + if (!rcv.parseArgs(new LinkedList<>(List.of(args))) && !rcv.helpOption) { System.exit(1); } if (rcv.helpOption) { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index 9c6613058f1..c81b8ff63b0 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -11,7 +11,6 @@ import java.io.BufferedOutputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -107,7 +106,7 @@ public class Arguments { private void parse(String[] argList) throws HelpShownException { List<String> args = new LinkedList<>(); - args.addAll(Arrays.asList(argList)); + args.addAll(List.of(argList)); while (!args.isEmpty()) { String arg = args.remove(0); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index 8182aa917ae..cd372e72e41 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -13,7 +13,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import java.io.InputStream; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Scanner; @@ -229,7 +228,7 @@ public class CommandLineOptions { private Iterator<String> getDocumentIds(CommandLine cl) { // Fetch document ids from stdin if no ids are passed in as command line arguments - List<String> documentIds = Arrays.asList(cl.getArgs()); + List<String> documentIds = List.of(cl.getArgs()); // WARNING: CommandLine.getArgs may return a single empty string as the only element if (documentIds.isEmpty() || documentIds.size() == 1 && documentIds.get(0).isEmpty()) { diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java index 2cb3ab17df4..56ca142841c 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -7,7 +7,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; -import java.util.Arrays; +import java.util.List; import com.yahoo.clientmetrics.RouteMetricSet; import com.yahoo.document.DocumentPut; @@ -39,7 +39,7 @@ public class VespaFeederTestCase { Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); FeederConfig config = arguments.getFeederConfig(); - assertEquals(false, config.abortondocumenterror()); + assertFalse(config.abortondocumenterror()); assertEquals(13.0, config.timeout(), 0.00001); assertEquals(false, config.retryenabled()); assertEquals("e6", config.route()); @@ -103,7 +103,7 @@ public class VespaFeederTestCase { public void assertRenderErrorOutput(String expected, String[] errors) { ArrayList<String> l = new ArrayList<String>(); - l.addAll(Arrays.asList(errors)); + l.addAll(List.of(errors)); assertEquals(expected, VespaFeeder.renderErrors(l).getMessage()); } @@ -111,16 +111,29 @@ public class VespaFeederTestCase { void testRenderErrors() { { String[] errors = {"foo"}; - assertRenderErrorOutput("Errors:\n" + - "-------\n" + - " foo\n", errors); + assertRenderErrorOutput(""" + Errors: + ------- + foo + """, errors); } { String[] errors = {"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}; - assertRenderErrorOutput("First 10 errors (of 11):\n" + - "------------------------\n" + - " 1\n 2\n 3\n 4\n 5\n 6\n 7\n 8\n 9\n 10\n", errors); + assertRenderErrorOutput(""" + First 10 errors (of 11): + ------------------------ + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + """, errors); } } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java index 0628f4e0f29..1d8f050a037 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -23,7 +23,6 @@ import org.mockito.ArgumentMatcher; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -99,7 +98,7 @@ public class DocumentRetrieverTest { } private static Iterator<String> asIterator(String... docIds) { - return Arrays.asList(docIds).iterator(); + return List.of(docIds).iterator(); } private static Reply createDocumentReply(String docId) { @@ -214,7 +213,7 @@ public class DocumentRetrieverTest { .setCluster(cluster) .build(); - ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef(cluster))); + ClusterList clusterList = new ClusterList(List.of(new ClusterDef(cluster))); DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); documentRetriever.retrieveDocuments(); @@ -230,7 +229,7 @@ public class DocumentRetrieverTest { .setCluster("invalidclustername") .build(); - ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef("storage"))); + ClusterList clusterList = new ClusterList(List.of(new ClusterDef("storage"))); DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); documentRetriever.retrieveDocuments(); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java index 143d5518eff..c4164d04aa2 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -28,7 +27,7 @@ public class BucketStatsPrinterTest { public void mockBucketStatsRetriever() throws BucketStatsException { retriever = mock(BucketStatsRetriever.class); when(retriever.getBucketIdForType(any(), any())).thenReturn(new BucketId(0x42)); - when(retriever.retrieveBucketList(any(), any())).thenReturn(Collections.emptyList()); + when(retriever.retrieveBucketList(any(), any())).thenReturn(List.of()); when(retriever.retrieveBucketStats(any(), any(), any(), any())).thenReturn(""); } @@ -79,7 +78,7 @@ public class BucketStatsPrinterTest { void testShouldPrintBucketStats() throws BucketStatsException { String dummyBucketStats = "dummystats"; GetBucketListReply.BucketInfo bucketInfo = new GetBucketListReply.BucketInfo(new BucketId(0), "dummy"); - when(retriever.retrieveBucketList(any(), any())).thenReturn(Collections.singletonList(bucketInfo)); + when(retriever.retrieveBucketList(any(), any())).thenReturn(List.of(bucketInfo)); when(retriever.retrieveBucketStats(any(), any(), any(), any())).thenReturn(dummyBucketStats); String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", true); diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java index 7f70256233e..1f215f9d90e 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java @@ -51,8 +51,7 @@ public class CollectionUtil { * Returns true if the contents of the two given collections are equal, ignoring order. */ public static boolean equalContentsIgnoreOrder(Collection<?> c1, Collection<?> c2) { - return c1.size() == c2.size() && - c1.containsAll(c2); + return c1.size() == c2.size() && c1.containsAll(c2); } /** diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java index 8ace7598e27..26c0bec34b1 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java @@ -56,7 +56,7 @@ public class ListenableArrayList<ITEM> extends ArrayList<ITEM> { } public List<Runnable> listeners() { - if (listeners == null) return Collections.emptyList(); + if (listeners == null) return List.of(); return Collections.unmodifiableList(listeners); } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java b/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java index 96541e555b2..63653b051c0 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/InThreadExecutorService.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; -import java.util.Collections; import java.util.List; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.ExecutorService; @@ -24,7 +23,7 @@ public class InThreadExecutorService extends AbstractExecutorService { @Override public List<Runnable> shutdownNow() { shutdown(); - return Collections.emptyList(); + return List.of(); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java index 3a02d752aa5..e67c1ebb3b6 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java @@ -41,8 +41,8 @@ public class Value implements Inspector { public void traverse(ObjectTraverser ot) {} public Inspector entry(int idx) { return invalid; } public Inspector field(java.lang.String name) { return invalid; } - public Iterable<Inspector> entries() { return Collections.emptyList(); } - public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { return Collections.emptyList(); } + public Iterable<Inspector> entries() { return List.of(); } + public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { return List.of(); } public StringBuilder writeJson(StringBuilder target) { return JsonRender.render(this, target, true); } diff --git a/vespajlib/src/main/java/com/yahoo/net/URI.java b/vespajlib/src/main/java/com/yahoo/net/URI.java index d008c589824..5111ad39e88 100644 --- a/vespajlib/src/main/java/com/yahoo/net/URI.java +++ b/vespajlib/src/main/java/com/yahoo/net/URI.java @@ -80,7 +80,7 @@ public class URI implements Cloneable, Comparable<URI> { } /** - * Creates an URI, optionaly keeping the fragment (the part starting by #). + * Creates a URI, optionally keeping the fragment (the part starting by #). * If the uri is hierarchical, it is normalized and incorrect hierarchical uris * which looks like urls are attempted repaired. * @@ -95,7 +95,7 @@ public class URI implements Cloneable, Comparable<URI> { } /** - * Creates an URI, optionaly keeping the fragment (the part starting by #). + * Creates a URI, optionally keeping the fragment (the part starting by #). * If the uri is hierarchical, it is normalized and incorrect hierarchical uris * which looks like urls are attempted repaired. * diff --git a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java index 704fe9d56c3..12b435111e3 100644 --- a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java +++ b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java @@ -39,12 +39,12 @@ public class CommandLineParser { } public CommandLineParser(String[] cmds) { - inputStrings = Arrays.asList(cmds); + inputStrings = List.of(cmds); } public CommandLineParser(String progname, String[] cmds) { this.progname=progname; - inputStrings = Arrays.asList(cmds); + inputStrings = List.of(cmds); } /** diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index d27c7cf0168..d225d6f4641 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -23,8 +23,6 @@ import com.yahoo.tensor.functions.Expand; import com.yahoo.tensor.impl.Label; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -170,7 +168,7 @@ public interface Tensor { /** Aggregates cells over a set of dimensions, or over all dimensions if no dimensions are specified */ default Tensor reduce(Reduce.Aggregator aggregator, String ... dimensions) { - return new Reduce<>(new ConstantTensor<>(this), aggregator, Arrays.asList(dimensions)).evaluate(); + return new Reduce<>(new ConstantTensor<>(this), aggregator, List.of(dimensions)).evaluate(); } /** Aggregates cells over a set of dimensions, or over all dimensions if no dimensions are specified */ default Tensor reduce(Reduce.Aggregator aggregator, List<String> dimensions) { @@ -186,8 +184,8 @@ public interface Tensor { } default Tensor rename(String fromDimension, String toDimension) { - return new Rename<>(new ConstantTensor<>(this), Collections.singletonList(fromDimension), - Collections.singletonList(toDimension)).evaluate(); + return new Rename<>(new ConstantTensor<>(this), List.of(fromDimension), + List.of(toDimension)).evaluate(); } default Tensor concat(double argument, String dimension) { @@ -270,26 +268,26 @@ public interface Tensor { default Tensor bit(Tensor argument) { return join(argument, (a,b) -> ((int)b < 8 && (int)b >= 0 && ((int)a & (1 << (int)b)) != 0) ? 1.0 : 0.0); } default Tensor hamming(Tensor argument) { return join(argument, Hamming::hamming); } - default Tensor avg() { return avg(Collections.emptyList()); } - default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); } + default Tensor avg() { return avg(List.of()); } + default Tensor avg(String dimension) { return avg(List.of(dimension)); } default Tensor avg(List<String> dimensions) { return reduce(Reduce.Aggregator.avg, dimensions); } - default Tensor count() { return count(Collections.emptyList()); } - default Tensor count(String dimension) { return count(Collections.singletonList(dimension)); } + default Tensor count() { return count(List.of()); } + default Tensor count(String dimension) { return count(List.of(dimension)); } default Tensor count(List<String> dimensions) { return reduce(Reduce.Aggregator.count, dimensions); } - default Tensor max() { return max(Collections.emptyList()); } - default Tensor max(String dimension) { return max(Collections.singletonList(dimension)); } + default Tensor max() { return max(List.of()); } + default Tensor max(String dimension) { return max(List.of(dimension)); } default Tensor max(List<String> dimensions) { return reduce(Reduce.Aggregator.max, dimensions); } - default Tensor median() { return median(Collections.emptyList()); } - default Tensor median(String dimension) { return median(Collections.singletonList(dimension)); } + default Tensor median() { return median(List.of()); } + default Tensor median(String dimension) { return median(List.of(dimension)); } default Tensor median(List<String> dimensions) { return reduce(Reduce.Aggregator.median, dimensions); } - default Tensor min() { return min(Collections.emptyList()); } - default Tensor min(String dimension) { return min(Collections.singletonList(dimension)); } + default Tensor min() { return min(List.of()); } + default Tensor min(String dimension) { return min(List.of(dimension)); } default Tensor min(List<String> dimensions) { return reduce(Reduce.Aggregator.min, dimensions); } - default Tensor prod() { return prod(Collections.emptyList()); } - default Tensor prod(String dimension) { return prod(Collections.singletonList(dimension)); } + default Tensor prod() { return prod(List.of()); } + default Tensor prod(String dimension) { return prod(List.of(dimension)); } default Tensor prod(List<String> dimensions) { return reduce(Reduce.Aggregator.prod, dimensions); } - default Tensor sum() { return sum(Collections.emptyList()); } - default Tensor sum(String dimension) { return sum(Collections.singletonList(dimension)); } + default Tensor sum() { return sum(List.of()); } + default Tensor sum(String dimension) { return sum(List.of(dimension)); } default Tensor sum(List<String> dimensions) { return reduce(Reduce.Aggregator.sum, dimensions); } // ----------------- non-math query methods (that is, computations not returning a tensor) diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java index b6bd252f135..fc82be4b33c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java @@ -3,7 +3,6 @@ package com.yahoo.tensor; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; @@ -50,7 +49,7 @@ public class TensorTypeParser { dimensionsSpec = specBody.substring(parenthesisIndex + 1); } - if (dimensionsSpec.isEmpty()) return new TensorType.Builder(valueType, Collections.emptyList()).build(); + if (dimensionsSpec.isEmpty()) return new TensorType.Builder(valueType, List.of()).build(); List<TensorType.Dimension> dimensions = new ArrayList<>(); for (String element : dimensionsSpec.split(",")) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java index 0cef1482292..58b17758cfe 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java @@ -7,7 +7,6 @@ import com.yahoo.tensor.functions.PrimitiveTensorFunction; import com.yahoo.tensor.functions.TensorFunction; import com.yahoo.tensor.functions.ToStringContext; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -34,7 +33,7 @@ public class VariableTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { return this; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java index 61207840ded..5655bb020a4 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java @@ -8,7 +8,6 @@ import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -32,7 +31,7 @@ public class CellCast<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java index c81cde70c75..0ecd4b5f947 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java @@ -7,7 +7,6 @@ import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -29,7 +28,7 @@ public class ConstantTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java index 382ac94be7d..354e03e6699 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java @@ -4,7 +4,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -27,7 +26,7 @@ public class Diag<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYP } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java index 3042991e2c0..f5a33dde064 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java @@ -4,7 +4,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -25,7 +24,7 @@ public class Expand<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java index 51bd4152479..a5afeb6d2a4 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java @@ -3,7 +3,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -21,7 +20,7 @@ public class L1Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java index 4b6ffbca63c..47e341732ca 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java @@ -3,7 +3,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -21,7 +20,7 @@ public class L2Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java index 404be1d6fac..94e75588ba3 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java @@ -9,7 +9,6 @@ import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -40,7 +39,7 @@ public class Map<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE public DoubleUnaryOperator mapper() { return mapper; } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/MapSubspaces.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/MapSubspaces.java index aa9602339e9..93a101909a2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/MapSubspaces.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/MapSubspaces.java @@ -9,7 +9,6 @@ import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -64,7 +63,7 @@ public class MapSubspaces<NAMETYPE extends Name> extends PrimitiveTensorFunction public TensorFunction<NAMETYPE> argument() { return argument; } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java index 5d51d8cd5c6..811e861de9c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java @@ -4,7 +4,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -24,7 +23,7 @@ public class Random<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java index 8520aef537d..9960fbd58ea 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java @@ -4,7 +4,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -28,7 +27,7 @@ public class Range<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETY } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java index c2eff01c801..150bf82f0e8 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java @@ -4,7 +4,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -26,7 +25,7 @@ public class Softmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAME } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(argument); } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { diff --git a/vespajlib/src/main/java/com/yahoo/yolean/trace/TraceNode.java b/vespajlib/src/main/java/com/yahoo/yolean/trace/TraceNode.java index 39a78897bfb..90b9383eb0c 100644 --- a/vespajlib/src/main/java/com/yahoo/yolean/trace/TraceNode.java +++ b/vespajlib/src/main/java/com/yahoo/yolean/trace/TraceNode.java @@ -3,7 +3,6 @@ package com.yahoo.yolean.trace; import com.yahoo.yolean.concurrent.ThreadRobustList; -import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -73,7 +72,7 @@ public class TraceNode { */ public <PAYLOADTYPE> Iterable<PAYLOADTYPE> descendants(final Class<PAYLOADTYPE> payloadType) { if (children == null) { - return Collections.emptyList(); + return List.of(); } return new Iterable<PAYLOADTYPE>() { @@ -118,7 +117,7 @@ public class TraceNode { */ public Iterable<TraceNode> children() { if (children == null) { - return Collections.emptyList(); + return List.of(); } return children; } diff --git a/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java index 54101c7f251..a2420ed3fe5 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.collections; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -14,16 +13,16 @@ import static org.junit.Assert.assertEquals; public class CollectionComparatorTestCase { @Test public void arrayLength() { - List<String> shortArr = Arrays.asList("x", "y"); - List<String> longArr = Arrays.asList("a", "b", "c", "d", "e"); + List<String> shortArr = List.of("x", "y"); + List<String> longArr = List.of("a", "b", "c", "d", "e"); assertEquals(-1, CollectionComparator.compare(shortArr, longArr)); } @Test public void compareArrays() { - List<String> one = Arrays.asList("b", "c", "d", "d", "e"); - List<String> two = Arrays.asList("a", "b", "c", "d", "e"); + List<String> one = List.of("b", "c", "d", "d", "e"); + List<String> two = List.of("a", "b", "c", "d", "e"); assertEquals(1, CollectionComparator.compare(one, two)); assertEquals(-1, CollectionComparator.compare(two, one)); @@ -31,8 +30,8 @@ public class CollectionComparatorTestCase { @Test public void compareEqualArrays() { - List<String> one = Arrays.asList("a", "b", "c", "d", "e"); - List<String> two = Arrays.asList("a", "b", "c", "d", "e"); + List<String> one = List.of("a", "b", "c", "d", "e"); + List<String> two = List.of("a", "b", "c", "d", "e"); assertEquals(0, CollectionComparator.compare(one, two)); assertEquals(0, CollectionComparator.compare(two, one)); diff --git a/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java b/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java index 7cef3423aeb..fd24bfb4f40 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java +++ b/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java @@ -27,7 +27,7 @@ public class CollectionUtilTest { @Test public void testMkString() { assertEquals("1, 2, 3, 4", - CollectionUtil.mkString(Arrays.asList(1, 2, 3, 4), ", ")); + CollectionUtil.mkString(List.of(1, 2, 3, 4), ", ")); } @Test @@ -35,16 +35,14 @@ public class CollectionUtilTest { List<Integer> l2Copy = new ArrayList<>(); l2Copy.addAll(l2); shuffle(); - assertTrue(CollectionUtil.equalContentsIgnoreOrder( - l2, l2Copy)); - assertFalse(CollectionUtil.equalContentsIgnoreOrder( - l1, l2)); + assertTrue(CollectionUtil.equalContentsIgnoreOrder(l2, l2Copy)); + assertFalse(CollectionUtil.equalContentsIgnoreOrder(l1, l2)); } @Test public void testSymmetricDifference() { assertTrue(CollectionUtil.equalContentsIgnoreOrder( - Arrays.asList(1, 2, 3), + List.of(1, 2, 3), CollectionUtil.symmetricDifference(l1, l2))); } } diff --git a/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java b/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java index 940dc159a17..930b274ef3e 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java +++ b/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java @@ -4,7 +4,6 @@ package com.yahoo.collections; import org.junit.Test; import org.mockito.Mockito; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -22,7 +21,6 @@ import static org.junit.Assert.fail; * @author Simon Thoresen Hult */ public class LazyMapTest { - @Test public void requireThatInitialDelegateIsEmpty() { LazyMap<String, String> map = newLazyMap(new HashMap<String, String>()); @@ -36,14 +34,14 @@ public class LazyMapTest { assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); map = newLazyMap(new HashMap<String, String>()); - map.putAll(Collections.singletonMap("foo", "bar")); + map.put("foo", "bar"); assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); } @Test public void requireThatEmptyMapPutAllEmptyMapDoesNotUpgradeToSingletonMap() { LazyMap<String, String> map = newLazyMap(new HashMap<String, String>()); - map.putAll(Collections.<String, String>emptyMap()); + map.putAll(Map.of()); assertEquals(LazyMap.EmptyMap.class, map.getDelegate().getClass()); } @@ -81,7 +79,7 @@ public class LazyMapTest { assertEquals("bar", map.put("foo", "baz")); assertEquals("baz", map.get("foo")); assertSame(delegate, map.getDelegate()); - map.putAll(Collections.singletonMap("foo", "cox")); + map.put("foo", "cox"); assertSame(delegate, map.getDelegate()); assertEquals("cox", map.get("foo")); } @@ -89,7 +87,7 @@ public class LazyMapTest { @Test public void requireThatSingletonMapPutAllEmptyMapDoesNotUpgradeToFinalMap() { LazyMap<String, String> map = newSingletonMap("foo", "bar"); - map.putAll(Collections.<String, String>emptyMap()); + map.putAll(Map.of()); assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); } @@ -188,7 +186,7 @@ public class LazyMapTest { Mockito.verify(delegate).put("foo", "bar"); Mockito.verify(delegate).put("baz", "cox"); - Map<String, String> arg = Collections.singletonMap("baz", "cox"); + Map<String, String> arg = Map.of("baz", "cox"); map.putAll(arg); Mockito.verify(delegate).putAll(arg); diff --git a/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java b/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java index ecefd891ca9..4acd9ef3214 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java +++ b/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java @@ -4,11 +4,11 @@ package com.yahoo.collections; import org.junit.Test; import org.mockito.Mockito; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Set; @@ -26,25 +26,25 @@ public class LazySetTest { @Test public void requireThatInitialDelegateIsEmpty() { - LazySet<String> set = newLazySet(new HashSet<String>()); + LazySet<String> set = newLazySet(new HashSet<>()); assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); } @Test public void requireThatEmptySetAddUpgradesToSingletonSet() { - LazySet<String> set = newLazySet(new HashSet<String>()); + LazySet<String> set = newLazySet(new HashSet<>()); assertTrue(set.add("foo")); assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); - set = newLazySet(new HashSet<String>()); - assertTrue(set.addAll(Arrays.asList("foo"))); + set = newLazySet(new HashSet<>()); + assertTrue(set.addAll(List.of("foo"))); assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); } @Test public void requireThatEmptySetAddAllEmptySetDoesNotUpgradeToSingletonSet() { - LazySet<String> set = newLazySet(new HashSet<String>()); - assertFalse(set.addAll(Collections.<String>emptySet())); + LazySet<String> set = newLazySet(new HashSet<>()); + assertFalse(set.addAll(Set.of())); assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); } @@ -52,7 +52,7 @@ public class LazySetTest { public void requireThatEmptySetAddAllUpgradesToFinalSet() { Set<String> delegate = new HashSet<>(); LazySet<String> set = newLazySet(delegate); - assertTrue(set.addAll(Arrays.asList("foo", "bar"))); + assertTrue(set.addAll(List.of("foo", "bar"))); assertSame(delegate, set.getDelegate()); assertEquals(2, delegate.size()); assertTrue(delegate.contains("foo")); @@ -76,7 +76,7 @@ public class LazySetTest { @Test public void requireThatSingletonSetAddAllEmptySetDoesNotUpgradeToFinalSet() { LazySet<String> set = newSingletonSet("foo"); - assertFalse(set.addAll(Collections.<String>emptySet())); + assertFalse(set.addAll(Set.of())); assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); } @@ -102,7 +102,7 @@ public class LazySetTest { public void requireThatSingletonSetAddAllUpgradesToFinalSet() { Set<String> delegate = new HashSet<>(); LazySet<String> set = newSingletonSet(delegate, "foo"); - assertTrue(set.addAll(Arrays.asList("bar"))); + assertTrue(set.addAll(List.of("bar"))); assertSame(delegate, set.getDelegate()); assertEquals(2, delegate.size()); assertTrue(delegate.contains("foo")); @@ -110,7 +110,7 @@ public class LazySetTest { delegate = new HashSet<>(); set = newSingletonSet(delegate, "foo"); - assertTrue(set.addAll(Arrays.asList("bar", "baz"))); + assertTrue(set.addAll(List.of("bar", "baz"))); assertSame(delegate, set.getDelegate()); assertEquals(3, delegate.size()); assertTrue(delegate.contains("foo")); @@ -198,15 +198,15 @@ public class LazySetTest { assertFalse(set.remove("foo")); Mockito.verify(delegate).remove("foo"); - Collection<String> containsAllArg = Collections.singletonList("foo"); + Collection<String> containsAllArg = Set.of("foo"); assertFalse(set.containsAll(containsAllArg)); Mockito.verify(delegate).containsAll(containsAllArg); - Collection<String> retainAllArg = Collections.singletonList("foo"); + Collection<String> retainAllArg = Set.of("foo"); assertFalse(set.retainAll(retainAllArg)); Mockito.verify(delegate).retainAll(retainAllArg); - Collection<String> removeAllArg = Collections.singletonList("foo"); + Collection<String> removeAllArg = Set.of("foo"); assertFalse(set.removeAll(removeAllArg)); Mockito.verify(delegate).removeAll(removeAllArg); diff --git a/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java index 536e4d62502..adee795bb48 100644 --- a/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.collections; import org.junit.Test; import static org.junit.Assert.*; -import java.util.Arrays; +import java.util.List; import java.util.ListIterator; /** @@ -22,9 +22,9 @@ public class ListenableArrayListTestCase { assertEquals(1,listener.invoked); list.add(0,"b"); assertEquals(2,listener.invoked); - list.addAll(Arrays.asList(new String[]{"c", "d"})); + list.addAll(List.of(new String[]{"c", "d"})); assertEquals(3,listener.invoked); - list.addAll(1,Arrays.asList(new String[]{"e", "f"})); + list.addAll(1,List.of(new String[]{"e", "f"})); assertEquals(4,listener.invoked); list.set(0,"g"); assertEquals(5,listener.invoked); diff --git a/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java index b1061fbc9ef..968746319d3 100644 --- a/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java @@ -6,7 +6,6 @@ import java.io.Reader; import java.io.StringReader; import java.lang.reflect.Method; import java.nio.CharBuffer; -import java.util.Collections; import java.util.List; import com.yahoo.protect.ClassValidator; @@ -30,7 +29,7 @@ public class NamedReaderTestCase { assertEquals("test1",r.getName()); assertEquals("test1",r.toString()); assertEquals(stringReader,r.getReader()); - NamedReader.closeAll(Collections.singletonList(r)); + NamedReader.closeAll(List.of(r)); NamedReader.closeAll(null); // noop, nor exception } diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java index 385d58db86f..100507c6cc1 100644 --- a/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java @@ -359,7 +359,7 @@ public class UrlTokenizerTestCase { } private static void assertTokenize(Url url, UrlToken... expected) { - Iterator<UrlToken> expectedIt = Arrays.asList(expected).iterator(); + Iterator<UrlToken> expectedIt = List.of(expected).iterator(); Iterator<UrlToken> actualIt = new UrlTokenizer(url).tokenize().iterator(); while (expectedIt.hasNext()) { assertTrue(actualIt.hasNext()); @@ -373,7 +373,7 @@ public class UrlTokenizerTestCase { List<UrlToken> actual = new LinkedList<>(); UrlTokenizer.addTokens(actual, UrlToken.Type.PATH, 0, img, true); - Iterator<String> expectedIt = Arrays.asList(expected).iterator(); + Iterator<String> expectedIt = List.of(expected).iterator(); Iterator<UrlToken> actualIt = actual.iterator(); while (expectedIt.hasNext()) { assertTrue(actualIt.hasNext()); diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java index 528ca57d256..b5d36326774 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java @@ -3,9 +3,9 @@ package com.yahoo.tensor; import org.junit.Test; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -139,7 +139,7 @@ public class IndexedTensorTestCase { Tensor.Builder builder = Tensor.Builder.of(type); builder.cell(47.0, 98); Tensor tensor = builder.build(); - assertEquals(47.0, tensor.sum(Collections.singletonList("x")).asDouble(), 0.000001); + assertEquals(47.0, tensor.sum(List.of("x")).asDouble(), 0.000001); } private void assertBuildingVWXYZ(TensorType type) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java index 378e2397a89..6b0dbac8f2a 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MatrixDotProductBenchmark.java @@ -9,7 +9,6 @@ import com.yahoo.tensor.functions.Join; import com.yahoo.tensor.functions.Reduce; import com.yahoo.tensor.functions.TensorFunction; -import java.util.Collections; import java.util.List; import java.util.Random; @@ -69,7 +68,7 @@ public class MatrixDotProductBenchmark { .value(random.nextDouble()); } } - return Collections.singletonList(builder.build()); + return List.of(builder.build()); } private static void addDimension(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 69fb71b9b0e..7a51ea300f4 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -11,7 +11,6 @@ import com.yahoo.tensor.functions.TensorFunction; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.function.DoubleBinaryOperator; @@ -214,7 +213,7 @@ public class TensorTestCase { Tensor unitK = Tensor.Builder.of(new TensorType.Builder().mapped("k").build()).cell().label("k", 0).value(1).build(); Tensor vectorInJSpace = vector(Type.mapped).multiply(unitJ); Tensor matrixInKSpace = matrix(Type.mapped, 2).get(0).multiply(unitK); - assertEquals("Generic computation implementation", 42, (int)dotProduct(vectorInJSpace, Collections.singletonList(matrixInKSpace))); + assertEquals("Generic computation implementation", 42, (int)dotProduct(vectorInJSpace, List.of(matrixInKSpace))); } @Test @@ -458,7 +457,7 @@ public class TensorTestCase { .value((i+1)*(j+1)); } } - return Collections.singletonList(builder.build()); + return List.of(builder.build()); } private TensorType vectorType(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java index 5d267d4d2f1..ce17a5d846d 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TypeResolverTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.tensor; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.junit.Test; @@ -17,7 +16,7 @@ import static org.junit.Assert.assertTrue; public class TypeResolverTestCase { private static List<String> mkl(String ...values) { - return Arrays.asList(values); + return List.of(values); } @Test diff --git a/vespalib/src/tests/fuzzy/fuzzy_matcher_test.cpp b/vespalib/src/tests/fuzzy/fuzzy_matcher_test.cpp index d94120e5bcf..9e4550db073 100644 --- a/vespalib/src/tests/fuzzy/fuzzy_matcher_test.cpp +++ b/vespalib/src/tests/fuzzy/fuzzy_matcher_test.cpp @@ -33,7 +33,7 @@ TEST(FuzzyMatcherTest, get_suffix_edge_cases) { } TEST(FuzzyMatcherTest, fuzzy_match_empty_prefix) { - FuzzyMatcher fuzzy("abc", 2, 0, false); + FuzzyMatcher fuzzy("abc", 2, 0, false, false); EXPECT_TRUE(fuzzy.isMatch("abc")); EXPECT_TRUE(fuzzy.isMatch("ABC")); EXPECT_TRUE(fuzzy.isMatch("ab1")); @@ -42,15 +42,15 @@ TEST(FuzzyMatcherTest, fuzzy_match_empty_prefix) { } TEST(FuzzyMatcherTest, fuzzy_match_cased) { - FuzzyMatcher fuzzy("abc", 2, 0, true); + FuzzyMatcher fuzzy("abc", 2, 0, true, false); EXPECT_TRUE(fuzzy.isMatch("abc")); EXPECT_TRUE(fuzzy.isMatch("abC")); EXPECT_TRUE(fuzzy.isMatch("aBC")); EXPECT_FALSE(fuzzy.isMatch("ABC")); } -TEST(FuzzyMatcherTest, fuzzy_match_with_prefix) { - FuzzyMatcher fuzzy("abcdef", 2, 2, false); +TEST(FuzzyMatcherTest, fuzzy_match_with_prefix_locking) { + FuzzyMatcher fuzzy("abcdef", 2, 2, false, false); EXPECT_TRUE(fuzzy.isMatch("abcdef")); EXPECT_TRUE(fuzzy.isMatch("ABCDEF")); EXPECT_TRUE(fuzzy.isMatch("abcde1")); @@ -59,22 +59,43 @@ TEST(FuzzyMatcherTest, fuzzy_match_with_prefix) { EXPECT_FALSE(fuzzy.isMatch("12cdef")); } -TEST(FuzzyMatcherTest, get_prefix_is_empty) { - FuzzyMatcher fuzzy("whatever", 2, 0, false); +TEST(FuzzyMatcherTest, get_prefix_lock_length_is_zero) { + FuzzyMatcher fuzzy("whatever", 2, 0, false, false); EXPECT_EQ(fuzzy.getPrefix(), ""); } TEST(FuzzyMatcherTest, term_is_empty) { - FuzzyMatcher fuzzy("", 2, 0, false); + FuzzyMatcher fuzzy("", 2, 0, false, false); EXPECT_TRUE(fuzzy.isMatch("")); EXPECT_TRUE(fuzzy.isMatch("a")); EXPECT_TRUE(fuzzy.isMatch("aa")); EXPECT_FALSE(fuzzy.isMatch("aaa")); } -TEST(FuzzyMatcherTest, get_prefix_non_empty) { - FuzzyMatcher fuzzy("abcd", 2, 2, false); +TEST(FuzzyMatcherTest, get_prefix_lock_length_non_zero) { + FuzzyMatcher fuzzy("abcd", 2, 2, false, false); EXPECT_EQ(fuzzy.getPrefix(), "ab"); } +TEST(FuzzyMatcherTest, fuzzy_prefix_matching_without_prefix_lock_length) { + FuzzyMatcher fuzzy("abc", 1, 0, false, true); + EXPECT_EQ(fuzzy.getPrefix(), ""); + EXPECT_TRUE(fuzzy.isMatch("abc")); + EXPECT_TRUE(fuzzy.isMatch("abcdefgh")); + EXPECT_TRUE(fuzzy.isMatch("ab")); + EXPECT_TRUE(fuzzy.isMatch("abd")); + EXPECT_TRUE(fuzzy.isMatch("xabc")); + EXPECT_FALSE(fuzzy.isMatch("xy")); +} + +TEST(FuzzyMatcherTest, fuzzy_prefix_matching_with_prefix_lock_length) { + FuzzyMatcher fuzzy("zoid", 1, 2, false, true); + EXPECT_EQ(fuzzy.getPrefix(), "zo"); + EXPECT_TRUE(fuzzy.isMatch("zoidberg")); + EXPECT_TRUE(fuzzy.isMatch("zold")); + EXPECT_TRUE(fuzzy.isMatch("zoldberg")); + EXPECT_FALSE(fuzzy.isMatch("zoxx")); + EXPECT_FALSE(fuzzy.isMatch("loid")); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp b/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp index 69b34ece2c7..dfd1b3b0c3b 100644 --- a/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp +++ b/vespalib/src/tests/fuzzy/levenshtein_dfa_test.cpp @@ -24,12 +24,13 @@ using CasingAndDfaType = std::tuple<LevenshteinDfa::Casing, LevenshteinDfa::DfaT namespace { std::optional<uint32_t> do_calculate(std::string_view left, std::string_view right, uint32_t threshold, - LevenshteinDfa::Casing casing, LevenshteinDfa::DfaType dfa_type) + LevenshteinDfa::Casing casing, LevenshteinDfa::DfaType dfa_type, + LevenshteinDfa::Matching matching) { - auto dfa_lhs = LevenshteinDfa::build(left, threshold, casing, dfa_type); + auto dfa_lhs = LevenshteinDfa::build(left, threshold, casing, dfa_type, matching); auto maybe_match_lhs = dfa_lhs.match(right); - auto dfa_rhs = LevenshteinDfa::build(right, threshold, casing, dfa_type); + auto dfa_rhs = LevenshteinDfa::build(right, threshold, casing, dfa_type, matching); auto maybe_match_rhs = dfa_rhs.match(left); EXPECT_EQ(maybe_match_lhs.matches(), maybe_match_rhs.matches()); @@ -40,6 +41,12 @@ std::optional<uint32_t> do_calculate(std::string_view left, std::string_view rig return std::nullopt; } +std::optional<uint32_t> do_calculate(std::string_view left, std::string_view right, uint32_t threshold, + LevenshteinDfa::Casing casing, LevenshteinDfa::DfaType dfa_type) +{ + return do_calculate(left, right, threshold, casing, dfa_type, LevenshteinDfa::Matching::FullString); +} + std::optional<uint32_t> do_calculate(std::u8string_view left, std::u8string_view right, uint32_t threshold, LevenshteinDfa::Casing casing, LevenshteinDfa::DfaType dfa_type) { @@ -74,6 +81,16 @@ struct LevenshteinDfaTest : TestWithParam<CasingAndDfaType> { return do_calculate(left, right, threshold, casing(), dfa_type()); } + static std::optional<uint32_t> prefix_calculate(std::string_view left, std::string_view right, uint32_t threshold) { + // Prefix matching is not symmetric, cannot use do_calculate() as it implicitly checks this. + auto dfa = LevenshteinDfa::build(left, threshold, casing(), dfa_type(), LevenshteinDfa::Matching::Prefix); + auto maybe_match = dfa.match(right); + if (maybe_match.matches()) { + return {maybe_match.edits()}; + } + return std::nullopt; + } + }; // All the baseline DFA tests use lowercase only, so they should have the exact same outcome @@ -98,6 +115,7 @@ TEST_P(LevenshteinDfaTest, edge_cases_have_correct_edit_distance) { EXPECT_EQ(calculate("abc", "ab", max), std::optional{1}) << max; EXPECT_EQ(calculate("abc", "abcd", max), std::optional{1}) << max; EXPECT_EQ(calculate("a", "", max), std::optional{1}) << max; + EXPECT_EQ(calculate("", "", max), std::optional{0}) << max; } EXPECT_EQ(calculate("bc", "abcd", 2), std::optional{2}); EXPECT_EQ(calculate("ab", "abcd", 2), std::optional{2}); @@ -109,6 +127,7 @@ TEST_P(LevenshteinDfaTest, edge_cases_have_correct_edit_distance) { EXPECT_EQ(calculate("ab", "", 2), std::optional{2}); EXPECT_EQ(calculate("abc", "", 2), std::nullopt); EXPECT_EQ(calculate("abc", "123", 2), std::nullopt); + EXPECT_EQ(calculate("abcde", "xad", 2), std::nullopt); } TEST_P(LevenshteinDfaTest, distance_is_in_utf32_code_point_space) { @@ -124,6 +143,37 @@ TEST_P(LevenshteinDfaTest, distance_is_in_utf32_code_point_space) { EXPECT_EQ(calculate(u8"カラオケ", u8"カラoke", 2), std::nullopt); } +TEST_P(LevenshteinDfaTest, can_match_in_target_string_prefix_mode) { + for (auto max : {1, 2}) { + EXPECT_EQ(prefix_calculate("", "literally anything", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("", "", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("x", "", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abc", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abcd", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abcdef", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "ab", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("ac", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("acd", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xabcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("bc", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "acb", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "acdefg", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("acb", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abd", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abdcfgh", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abdefgh", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xbc", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xbcdefg", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xy", max), std::nullopt); + } + EXPECT_EQ(prefix_calculate("abc", "xxabc", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abc", "xxabcd", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abcxx", "abc", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abcxx", "abcd", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("xy", "", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("xyz", "", 2), std::nullopt); +} + void test_dfa_successor(const LevenshteinDfa& dfa, std::string_view source, std::string_view expected_successor, std::string_view successor_prefix) { @@ -182,11 +232,22 @@ TEST_P(LevenshteinDfaTest, can_generate_successors_to_mismatching_source_strings "\xf0\x9f\xa4\xa9""food"); // <starry eyed emoji>food // Note that as a general rule, emojis are fickle beasts to deal with since a single - // emoji often takes up multiple code points, which we consider separate characters - // but a user sees as a single actual rendered glyph. + // emoji often takes up multiple code points. We consider these as separate characters + // but from a Unicode perspective they are considered part of an "extended grapheme + // cluster", from which the actually rendered glyph is derived. // Multi-code point character edit distance support is left as an exercise for the reader :D } +TEST_P(LevenshteinDfaTest, can_generate_successors_with_prefix_match_mode) { + auto dfa = LevenshteinDfa::build("ban", 1, casing(), dfa_type(), LevenshteinDfa::Matching::Prefix); + // There is no difference at all in successor output when using Prefix mode as opposed to + // FullString mode, but since we match _more_ source strings in prefix mode we cannot + // simply run the same test set for both modes. This just tests that the wiring seems fine. + test_dfa_successor(dfa, "", "\x01""an"); + test_dfa_successor(dfa, "xy", "yan"); + test_dfa_successor(dfa, "bolle", "bon"); +} + TEST_P(LevenshteinDfaTest, successor_is_well_defined_for_max_unicode_code_point_input) { auto dfa = LevenshteinDfa::build("food", 1, casing(), dfa_type()); // The successor string must be lexicographically larger than the input string. @@ -332,13 +393,21 @@ std::string bits_to_str(T v) { return ret; } -using CasingAndDfaTypeAndMaxEdits = std::tuple<LevenshteinDfa::Casing, LevenshteinDfa::DfaType, uint32_t>; +using CasingAndDfaTypeAndMaxEdits = std::tuple< + LevenshteinDfa::Casing, + LevenshteinDfa::DfaType, + LevenshteinDfa::Matching, + uint32_t +>; struct LevenshteinDfaSuccessorTest : TestWithParam<CasingAndDfaTypeAndMaxEdits> { - // Print test suffix as e.g. "/Uncased_Explicit_1" instead of just a GTest-chosen number. + // Print test suffix as e.g. "/Uncased_Explicit_FullString_1" instead of just a GTest-chosen number. static std::string stringify_params(const TestParamInfo<ParamType>& info) { std::ostringstream ss; - ss << std::get<0>(info.param) << '_' << std::get<1>(info.param) << '_' << std::get<2>(info.param); + ss << std::get<0>(info.param) + << '_' << std::get<1>(info.param) + << '_' << std::get<2>(info.param) + << '_' << std::get<3>(info.param); return ss.str(); } }; @@ -350,6 +419,8 @@ INSTANTIATE_TEST_SUITE_P(SupportedMaxEdits, Values(LevenshteinDfa::DfaType::Explicit, LevenshteinDfa::DfaType::Implicit, LevenshteinDfa::DfaType::Table), + Values(LevenshteinDfa::Matching::FullString, + LevenshteinDfa::Matching::Prefix), Values(1, 2)), LevenshteinDfaSuccessorTest::stringify_params); @@ -369,10 +440,10 @@ INSTANTIATE_TEST_SUITE_P(SupportedMaxEdits, * Inspired by approach used by Lucene DFA exhaustive testing. */ TEST_P(LevenshteinDfaSuccessorTest, exhaustive_successor_test) { - const auto [casing, dfa_type, max_edits] = GetParam(); + const auto [casing, dfa_type, matching, max_edits] = GetParam(); for (uint32_t i = 0; i < 256; ++i) { const auto target = bits_to_str(static_cast<uint8_t>(i)); - auto target_dfa = LevenshteinDfa::build(target, max_edits, casing, dfa_type); + auto target_dfa = LevenshteinDfa::build(target, max_edits, casing, dfa_type, matching); std::string skip_to, successor; for (uint32_t j = 0; j < 256; ++j) { const auto source = bits_to_str(static_cast<uint8_t>(j)); diff --git a/vespalib/src/tests/fuzzy/levenshtein_distance_test.cpp b/vespalib/src/tests/fuzzy/levenshtein_distance_test.cpp index 37d1d4b4c54..61824fa4abf 100644 --- a/vespalib/src/tests/fuzzy/levenshtein_distance_test.cpp +++ b/vespalib/src/tests/fuzzy/levenshtein_distance_test.cpp @@ -16,6 +16,13 @@ std::optional<uint32_t> calculate(std::string_view left, std::string_view right, return leftRight; } +// Prefix matching is asymmetric and therefore cannot implicitly test result symmetry +std::optional<uint32_t> prefix_calculate(std::string_view left, std::string_view right, uint32_t threshold) { + auto left_codepoints = vespalib::LowerCase::convert_to_ucs4(left); + auto right_codepoints = vespalib::LowerCase::convert_to_ucs4(right); + return vespalib::LevenshteinDistance::calculate(left_codepoints, right_codepoints, threshold, true); +} + TEST(LevenshteinDistance, calculate_edgecases) { EXPECT_EQ(calculate("abc", "abc", 2), std::optional{0}); EXPECT_EQ(calculate("abc", "ab1", 2), std::optional{1}); @@ -33,6 +40,58 @@ TEST(LevenshteinDistance, calculate_edgecases) { EXPECT_EQ(calculate("ab", "", 2), std::optional{2}); EXPECT_EQ(calculate("abc", "", 2), std::nullopt); EXPECT_EQ(calculate("abc", "123", 2), std::nullopt); + EXPECT_EQ(calculate("abcde", "xad", 2), std::nullopt); +} + +TEST(LevenshteinDistance, prefix_match_edge_cases) { + // Same cases as LevenshteinDfaTest (TODO consolidate these somehow) + for (auto max : {1, 2}) { + EXPECT_EQ(prefix_calculate("", "literally anything", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("", "", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("x", "", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abc", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abcd", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abcdef", max), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "ab", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("ac", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("acd", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xabcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("bc", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "acb", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "acdefg", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("acb", "abcdef", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abd", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abdcfgh", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "abdefgh", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xbc", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xbcdefg", max), std::optional{1}); + EXPECT_EQ(prefix_calculate("abc", "xy", max), std::nullopt); + } + EXPECT_EQ(prefix_calculate("abc", "xxabc", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abc", "xxabcd", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abcxx", "abc", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("abcxx", "abcd", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("xy", "", 2), std::optional{2}); + EXPECT_EQ(prefix_calculate("xyz", "", 2), std::nullopt); + + // Max edits not in {1, 2} cases; not supported by DFA implementation. + EXPECT_EQ(prefix_calculate("", "", 0), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abc", 0), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "abcde", 0), std::optional{0}); + EXPECT_EQ(prefix_calculate("abc", "dbc", 0), std::nullopt); + EXPECT_EQ(prefix_calculate("abc", "", 3), std::optional{3}); + EXPECT_EQ(prefix_calculate("abc", "xy", 3), std::optional{3}); + EXPECT_EQ(prefix_calculate("abc", "xyz", 3), std::optional{3}); + EXPECT_EQ(prefix_calculate("abc", "xyzzz", 3), std::optional{3}); + EXPECT_EQ(prefix_calculate("abcd", "xyzd", 3), std::optional{3}); + EXPECT_EQ(prefix_calculate("abcd", "xyzz", 3), std::nullopt); + EXPECT_EQ(prefix_calculate("abcd", "", 3), std::nullopt); +} + +TEST(LevenshteinDistance, oversized_max_edits_is_well_defined) { + const auto k = uint32_t(INT32_MAX) + 10000u; + EXPECT_EQ(calculate("abc", "xyz", k), std::optional{3}); + EXPECT_EQ(prefix_calculate("abc", "xyzzzz", k), std::optional{3}); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/hamming/CMakeLists.txt b/vespalib/src/tests/util/hamming/CMakeLists.txt index 5c317627200..ab551eab583 100644 --- a/vespalib/src/tests/util/hamming/CMakeLists.txt +++ b/vespalib/src/tests/util/hamming/CMakeLists.txt @@ -7,3 +7,10 @@ vespa_add_executable(vespalib_hamming_test_app TEST GTest::GTest ) vespa_add_test(NAME vespalib_hamming_test_app COMMAND vespalib_hamming_test_app) + +vespa_add_executable(vespalib_hamming_benchmark_app TEST + SOURCES + hamming_benchmark.cpp + DEPENDS + vespalib +) diff --git a/vespalib/src/tests/util/hamming/hamming_benchmark.cpp b/vespalib/src/tests/util/hamming/hamming_benchmark.cpp new file mode 100644 index 00000000000..347c935f5b7 --- /dev/null +++ b/vespalib/src/tests/util/hamming/hamming_benchmark.cpp @@ -0,0 +1,41 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/binary_hamming_distance.h> +#include <vector> +#include <cinttypes> +#include <cstdlib> +#include <cstdint> +#include <cstdio> + +using namespace vespalib; + +int main(int argc, char* argv[]) { + size_t vector_length = 1024/8; + size_t num_vectors = 1; + size_t num_reps = 100000000; + + if (argc > 2) { + vector_length = atol(argv[2])/8; + } + if (argc > 3) { + num_reps = atol(argv[3]); + } + if (argc > 4) { + num_vectors = atol(argv[4]); + } + + std::vector<uint8_t> center(vector_length); + std::vector<uint8_t> vectors(num_vectors*vector_length); + srand(13); + for (uint8_t & v : center) { v = rand(); } + for (uint8_t & v : vectors) { v = rand(); } + uint64_t sum(0); + for (size_t i=0; i < num_reps; i++) { + for (size_t j(0); j < num_vectors; j++) { + sum += binary_hamming_distance(center.data(), vectors.data() + j*vector_length, vector_length); + } + } + + printf("%lu vectors of %lu bits, repeated %lu times. Sum of distances = %" PRIu64 "\n", num_vectors, vector_length*8, num_reps, sum); + return 0; +} diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index 4549b81283e..51a1f9fe950 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -94,7 +94,7 @@ private: EntryRef allocate_dynamic_array(size_t array_size, uint32_t type_id); EntryRef addLargeArray(ConstArrayRef array); EntryRef allocate_large_array(size_t array_size); - ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const { + ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const noexcept { const ElemT *buf = _store.template getEntryArray<ElemT>(ref, arraySize); return ConstArrayRef(buf, arraySize); } @@ -104,7 +104,7 @@ private: auto size = BufferType::get_dynamic_array_size(entry); return ConstArrayRef(entry, size); } - ConstArrayRef getLargeArray(RefT ref) const { + ConstArrayRef getLargeArray(RefT ref) const noexcept { const LargeArray *buf = _store.template getEntry<LargeArray>(ref); return ConstArrayRef(&(*buf)[0], buf->size()); } @@ -114,7 +114,7 @@ public: ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, TypeMapper&& mapper); ~ArrayStore() override; EntryRef add(ConstArrayRef array); - ConstArrayRef get(EntryRef ref) const { + ConstArrayRef get(EntryRef ref) const noexcept { if (!ref.valid()) [[unlikely]] { return ConstArrayRef(); } diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h index fa231e9cf94..0226c780cf1 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.h +++ b/vespalib/src/vespa/vespalib/datastore/datastore.h @@ -96,7 +96,7 @@ public: EntryRef addEntry(const EntryType &e); - const EntryType &getEntry(EntryRef ref) const { + const EntryType &getEntry(EntryRef ref) const noexcept { return *this->template getEntry<EntryType>(RefType(ref)); } }; diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp index f9552904cfb..674af23f46f 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp @@ -4,7 +4,7 @@ namespace vespalib::datastore { -MemoryStats::MemoryStats() +MemoryStats::MemoryStats() noexcept : _alloc_entries(0), _used_entries(0), _dead_entries(0), @@ -20,7 +20,7 @@ MemoryStats::MemoryStats() } MemoryStats& -MemoryStats::operator+=(const MemoryStats& rhs) +MemoryStats::operator+=(const MemoryStats& rhs) noexcept { _alloc_entries += rhs._alloc_entries; _used_entries += rhs._used_entries; diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.h b/vespalib/src/vespa/vespalib/datastore/memory_stats.h index a5988d00f6b..65066690e8c 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.h +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.h @@ -25,8 +25,8 @@ public: uint32_t _activeBuffers; uint32_t _holdBuffers; - MemoryStats(); - MemoryStats& operator+=(const MemoryStats& rhs); + MemoryStats() noexcept; + MemoryStats& operator+=(const MemoryStats& rhs) noexcept; }; } diff --git a/vespalib/src/vespa/vespalib/fuzzy/dfa_matcher.h b/vespalib/src/vespa/vespalib/fuzzy/dfa_matcher.h index c69f414aae6..8b2597dd179 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/dfa_matcher.h +++ b/vespalib/src/vespa/vespalib/fuzzy/dfa_matcher.h @@ -20,6 +20,12 @@ concept DfaMatcher = requires(T a, std::string u8str, std::vector<uint32_t> u32s // matching to have the expected semantics, the actual target string must be pre-lowercased. { a.is_cased() } -> std::same_as<bool>; + // Whether the matching is performed in prefix mode. In prefix mode, a source string is + // considered a match if it matches the target string at any point during DFA traversal. + // I.e. the source string "bananas" prefix-matched against the target (prefix) "ban" will + // be returned as a match with 0 edits. + { a.is_prefix() } -> std::same_as<bool>; + // Initial (starting) state of the DFA { a.start() } -> std::same_as<typename T::StateType>; diff --git a/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.h index 490582b5bf7..a3542d61dc8 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.h +++ b/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.h @@ -97,9 +97,11 @@ public: private: std::vector<DfaNodeType> _nodes; const bool _is_cased; + const bool _is_prefix; public: - explicit ExplicitLevenshteinDfaImpl(bool is_cased) noexcept - : _is_cased(is_cased) + ExplicitLevenshteinDfaImpl(bool is_cased, bool is_prefix) noexcept + : _is_cased(is_cased), + _is_prefix(is_prefix) {} ~ExplicitLevenshteinDfaImpl() override; @@ -140,10 +142,12 @@ template <typename Traits> class ExplicitLevenshteinDfaBuilder { const std::vector<uint32_t> _u32_str_buf; // TODO std::u32string const bool _is_cased; + const bool _is_prefix; public: - ExplicitLevenshteinDfaBuilder(std::vector<uint32_t> str, bool is_cased) noexcept + ExplicitLevenshteinDfaBuilder(std::vector<uint32_t> str, bool is_cased, bool is_prefix) noexcept : _u32_str_buf(std::move(str)), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) {} [[nodiscard]] LevenshteinDfa build_dfa() const; diff --git a/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.hpp b/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.hpp index 55dd459ff26..11a822d2d7e 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.hpp +++ b/vespalib/src/vespa/vespalib/fuzzy/explicit_levenshtein_dfa.hpp @@ -22,16 +22,20 @@ struct ExplicitDfaMatcher { const std::span<const DfaNodeType> _nodes; const bool _is_cased; + const bool _is_prefix; - ExplicitDfaMatcher(const std::span<const DfaNodeType> nodes, bool is_cased) noexcept + ExplicitDfaMatcher(const std::span<const DfaNodeType> nodes, bool is_cased, bool is_prefix) noexcept : _nodes(nodes), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) {} static constexpr uint8_t max_edits() noexcept { return MaxEdits; } bool is_cased() const noexcept { return _is_cased; } + bool is_prefix() const noexcept { return _is_prefix; } + StateType start() const noexcept { return &_nodes[0]; } @@ -99,21 +103,21 @@ ExplicitLevenshteinDfaImpl<MaxEdits>::~ExplicitLevenshteinDfaImpl() = default; template <uint8_t MaxEdits> LevenshteinDfa::MatchResult ExplicitLevenshteinDfaImpl<MaxEdits>::match(std::string_view u8str) const { - ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased); + ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased, _is_prefix); return MatchAlgorithm<MaxEdits>::match(matcher, u8str); } template <uint8_t MaxEdits> LevenshteinDfa::MatchResult ExplicitLevenshteinDfaImpl<MaxEdits>::match(std::string_view u8str, std::string& successor_out) const { - ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased); + ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased, _is_prefix); return MatchAlgorithm<MaxEdits>::match(matcher, u8str, successor_out); } template <uint8_t MaxEdits> LevenshteinDfa::MatchResult ExplicitLevenshteinDfaImpl<MaxEdits>::match(std::string_view u8str, std::vector<uint32_t>& successor_out) const { - ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased); + ExplicitDfaMatcher<MaxEdits> matcher(_nodes, _is_cased, _is_prefix); return MatchAlgorithm<MaxEdits>::match(matcher, u8str, successor_out); } @@ -199,10 +203,12 @@ class ExplicitLevenshteinDfaBuilderImpl : public DfaSteppingBase<Traits> { using Base::transitions; const bool _is_cased; + const bool _is_prefix; public: - ExplicitLevenshteinDfaBuilderImpl(std::span<const uint32_t> str, bool is_cased) noexcept + ExplicitLevenshteinDfaBuilderImpl(std::span<const uint32_t> str, bool is_cased, bool is_prefix) noexcept : DfaSteppingBase<Traits>(str), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) { assert(str.size() < UINT32_MAX / max_out_edges_per_node()); } @@ -217,7 +223,7 @@ public: template <typename Traits> LevenshteinDfa ExplicitLevenshteinDfaBuilderImpl<Traits>::build_dfa() const { - auto dfa = std::make_unique<ExplicitLevenshteinDfaImpl<max_edits()>>(_is_cased); + auto dfa = std::make_unique<ExplicitLevenshteinDfaImpl<max_edits()>>(_is_cased, _is_prefix); ExploreState<StateType> exp; // Use BFS instead of DFS to ensure most node edges point to nodes that are allocated _after_ // the parent node, which means the CPU can skip ahead instead of ping-ponging back and forth. @@ -257,7 +263,7 @@ LevenshteinDfa ExplicitLevenshteinDfaBuilderImpl<Traits>::build_dfa() const { template <typename Traits> LevenshteinDfa ExplicitLevenshteinDfaBuilder<Traits>::build_dfa() const { - ExplicitLevenshteinDfaBuilderImpl<Traits> builder(_u32_str_buf, _is_cased); + ExplicitLevenshteinDfaBuilderImpl<Traits> builder(_u32_str_buf, _is_cased, _is_prefix); return builder.build_dfa(); } diff --git a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.cpp b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.cpp index 71388f43b89..fbfa50aa57f 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.cpp +++ b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.cpp @@ -29,10 +29,12 @@ vespalib::FuzzyMatcher::FuzzyMatcher(): _folded_term_codepoints_suffix() {} -vespalib::FuzzyMatcher::FuzzyMatcher(std::string_view term, uint32_t max_edit_distance, uint32_t prefix_size, bool is_cased): +vespalib::FuzzyMatcher::FuzzyMatcher(std::string_view term, uint32_t max_edit_distance, uint32_t prefix_size, + bool is_cased, bool is_prefix): _max_edit_distance(max_edit_distance), _prefix_size(prefix_size), _is_cased(is_cased), + _is_prefix(is_prefix), _folded_term_codepoints(_is_cased ? cased_convert_to_ucs4(term) : LowerCase::convert_to_ucs4(term)), _folded_term_codepoints_prefix(get_prefix(_folded_term_codepoints, _prefix_size)), _folded_term_codepoints_suffix(get_suffix(_folded_term_codepoints, _prefix_size)) @@ -73,7 +75,7 @@ bool vespalib::FuzzyMatcher::isMatch(std::string_view target) const { return LevenshteinDistance::calculate( _folded_term_codepoints_suffix, get_suffix(targetCodepoints, _prefix_size), - _max_edit_distance).has_value(); + _max_edit_distance, _is_prefix).has_value(); } vespalib::string vespalib::FuzzyMatcher::getPrefix() const { diff --git a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.h b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.h index aae6ca1c6e5..2c17283d20a 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.h +++ b/vespalib/src/vespa/vespalib/fuzzy/fuzzy_matcher.h @@ -24,6 +24,7 @@ private: uint32_t _max_edit_distance; // max edit distance uint32_t _prefix_size; // prefix of a term that is considered frozen, i.e. non-fuzzy bool _is_cased; + bool _is_prefix; std::vector<uint32_t> _folded_term_codepoints; @@ -34,7 +35,7 @@ public: FuzzyMatcher(); FuzzyMatcher(const FuzzyMatcher &) = delete; FuzzyMatcher & operator = (const FuzzyMatcher &) = delete; - FuzzyMatcher(std::string_view term, uint32_t max_edit_distance, uint32_t prefix_size, bool is_cased); + FuzzyMatcher(std::string_view term, uint32_t max_edit_distance, uint32_t prefix_size, bool is_cased, bool is_prefix); ~FuzzyMatcher(); [[nodiscard]] bool isMatch(std::string_view target) const; diff --git a/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.h index 20c8e1b7e2b..552fd849734 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.h +++ b/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.h @@ -13,14 +13,16 @@ class ImplicitLevenshteinDfa final : public LevenshteinDfa::Impl { std::string _target_as_utf8; std::vector<uint32_t> _target_utf8_char_offsets; const bool _is_cased; + const bool _is_prefix; public: using MatchResult = LevenshteinDfa::MatchResult; - ImplicitLevenshteinDfa(std::vector<uint32_t> str, bool is_cased) + ImplicitLevenshteinDfa(std::vector<uint32_t> str, bool is_cased, bool is_prefix) : _u32_str_buf(std::move(str)), _target_as_utf8(), _target_utf8_char_offsets(), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) { precompute_utf8_target_with_offsets(); } diff --git a/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.hpp b/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.hpp index 05ef2761f34..7dadcc59b8e 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.hpp +++ b/vespalib/src/vespa/vespalib/fuzzy/implicit_levenshtein_dfa.hpp @@ -32,21 +32,26 @@ struct ImplicitDfaMatcher : public DfaSteppingBase<Traits> { std::span<const char> _target_as_utf8; std::span<const uint32_t> _target_utf8_char_offsets; const bool _is_cased; + const bool _is_prefix; ImplicitDfaMatcher(std::span<const uint32_t> u32_str, std::span<const char> target_as_utf8, std::span<const uint32_t> target_utf8_char_offsets, - bool is_cased) noexcept + bool is_cased, + bool is_prefix) noexcept : Base(u32_str), _target_as_utf8(target_as_utf8), _target_utf8_char_offsets(target_utf8_char_offsets), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) {} // start, is_match, can_match, match_edit_distance are all provided by base type bool is_cased() const noexcept { return _is_cased; } + bool is_prefix() const noexcept { return _is_prefix; } + template <typename F> bool has_any_char_matching(const StateType& state, F&& f) const noexcept(noexcept(f(uint32_t{}))) { for (uint32_t i = 0; i < state.size(); ++i) { @@ -137,21 +142,21 @@ struct ImplicitDfaMatcher : public DfaSteppingBase<Traits> { template <typename Traits> LevenshteinDfa::MatchResult ImplicitLevenshteinDfa<Traits>::match(std::string_view u8str) const { - ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased); + ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased, _is_prefix); return MatchAlgorithm<Traits::max_edits()>::match(matcher, u8str); } template <typename Traits> LevenshteinDfa::MatchResult ImplicitLevenshteinDfa<Traits>::match(std::string_view u8str, std::string& successor_out) const { - ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased); + ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased, _is_prefix); return MatchAlgorithm<Traits::max_edits()>::match(matcher, u8str, successor_out); } template <typename Traits> LevenshteinDfa::MatchResult ImplicitLevenshteinDfa<Traits>::match(std::string_view u8str, std::vector<uint32_t>& successor_out) const { - ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased); + ImplicitDfaMatcher<Traits> matcher(_u32_str_buf, _target_as_utf8, _target_utf8_char_offsets, _is_cased, _is_prefix); return MatchAlgorithm<Traits::max_edits()>::match(matcher, u8str, successor_out); } diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp index 19c2fffbb3e..468619c8036 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.cpp @@ -41,34 +41,40 @@ void LevenshteinDfa::dump_as_graphviz(std::ostream& out) const { _impl->dump_as_graphviz(out); } -LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max_edits, Casing casing, DfaType dfa_type) { +LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max_edits, + Casing casing, DfaType dfa_type, Matching matching) { if (max_edits != 1 && max_edits != 2) { throw std::invalid_argument(make_string("Levenshtein DFA max_edits must be in {1, 2}, was %u", max_edits)); } - const bool is_cased = (casing == Casing::Cased); + const bool is_cased = (casing == Casing::Cased); + const bool is_prefix = (matching == Matching::Prefix); auto target_string_u32 = is_cased ? utf8_string_to_utf32(target_string) : utf8_string_to_utf32_lowercased(target_string); if (dfa_type == DfaType::Implicit) { if (max_edits == 1) { - return LevenshteinDfa(std::make_unique<ImplicitLevenshteinDfa<FixedMaxEditDistanceTraits<1>>>(std::move(target_string_u32), is_cased)); + return LevenshteinDfa(std::make_unique<ImplicitLevenshteinDfa<FixedMaxEditDistanceTraits<1>>>(std::move(target_string_u32), is_cased, is_prefix)); } else { // max_edits == 2 - return LevenshteinDfa(std::make_unique<ImplicitLevenshteinDfa<FixedMaxEditDistanceTraits<2>>>(std::move(target_string_u32), is_cased)); + return LevenshteinDfa(std::make_unique<ImplicitLevenshteinDfa<FixedMaxEditDistanceTraits<2>>>(std::move(target_string_u32), is_cased, is_prefix)); } } else if(dfa_type == DfaType::Explicit) { if (max_edits == 1) { - return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<1>>(std::move(target_string_u32), is_cased).build_dfa(); + return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<1>>(std::move(target_string_u32), is_cased, is_prefix).build_dfa(); } else { // max_edits == 2 - return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<2>>(std::move(target_string_u32), is_cased).build_dfa(); + return ExplicitLevenshteinDfaBuilder<FixedMaxEditDistanceTraits<2>>(std::move(target_string_u32), is_cased, is_prefix).build_dfa(); } } else { // DfaType::Table if (max_edits == 1) { - return LevenshteinDfa(std::make_unique<TableDfa<1>>(std::move(target_string_u32), is_cased)); + return LevenshteinDfa(std::make_unique<TableDfa<1>>(std::move(target_string_u32), is_cased, is_prefix)); } else { // max_edits == 2 - return LevenshteinDfa(std::make_unique<TableDfa<2>>(std::move(target_string_u32), is_cased)); + return LevenshteinDfa(std::make_unique<TableDfa<2>>(std::move(target_string_u32), is_cased, is_prefix)); } } } +LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max_edits, Casing casing, DfaType dfa_type) { + return build(target_string, max_edits, casing, dfa_type, Matching::FullString); +} + LevenshteinDfa LevenshteinDfa::build(std::string_view target_string, uint8_t max_edits, Casing casing) { // TODO automatically select implementation based on target length/max edits? // Suggestion: @@ -112,4 +118,12 @@ std::ostream& operator<<(std::ostream& os, LevenshteinDfa::Casing c) { return os; } +std::ostream& operator<<(std::ostream& os, LevenshteinDfa::Matching m) { + switch (m) { + case LevenshteinDfa::Matching::FullString: os << "FullString"; return os; + case LevenshteinDfa::Matching::Prefix: os << "Prefix"; return os; + } + abort(); +} + } diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h index 44c62bdb957..feace39b313 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_dfa.h @@ -265,6 +265,33 @@ public: Cased }; + enum class Matching { + /** + * Edit distance is computed based on the _entire_ source string. Matching is + * symmetric between source and target strings, i.e. match(x, y) and match(y, x) + * will yield the same result. + */ + FullString, + /** + * Edit distance is computed based on a _prefix_ of the source string, as compared + * against the target string. Matching is therefore _asymmetric_ between source and + * target strings. + * + * Example of matching source strings against the target 'ban' (i.e. the prefix query) + * and 1 max edit distance: + * 'bananas' - 0 edits (source prefix 'ban' exact-matches 'ban') + * 'balloons' - 1 edit ('bal' vs 'ban') + * '2bananas' - 1 edit ('2ban' vs 'ban') + * 'boonanas' - mismatch (2 edits) + * + * Note that Prefix matching will match a lot more strings than FullString, so in + * practice it should be combined with prefix _locking_ to constrain the candidate + * result set to a more reasonable cardinality. In particular, max edits >= |target| + * will match _every_ source string trivially. + */ + Prefix + }; + /** * Builds and returns a Levenshtein DFA that matches all strings within `max_edits` * edits of `target_string`. The type of DFA returned is specified by dfa_type. @@ -274,6 +301,9 @@ public: * `target_string` must not contain any null UTF-8 chars. */ [[nodiscard]] static LevenshteinDfa build(std::string_view target_string, uint8_t max_edits, + Casing casing, DfaType dfa_type, Matching matching); + + [[nodiscard]] static LevenshteinDfa build(std::string_view target_string, uint8_t max_edits, Casing casing, DfaType dfa_type); /** @@ -301,5 +331,6 @@ public: std::ostream& operator<<(std::ostream& os, const LevenshteinDfa::MatchResult& mos); std::ostream& operator<<(std::ostream& os, LevenshteinDfa::DfaType dt); std::ostream& operator<<(std::ostream& os, LevenshteinDfa::Casing c); +std::ostream& operator<<(std::ostream& os, LevenshteinDfa::Matching m); } diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.cpp b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.cpp index 4658635d880..f5b8d0fc0b8 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.cpp +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.cpp @@ -3,31 +3,46 @@ #include "levenshtein_distance.h" +#include <cassert> #include <limits> #include <vector> +namespace vespalib { + std::optional<uint32_t> -vespalib::LevenshteinDistance::calculate(std::span<const uint32_t> left, std::span<const uint32_t> right, uint32_t threshold) +LevenshteinDistance::calculate(std::span<const uint32_t> left, std::span<const uint32_t> right, + uint32_t threshold, bool prefix_match) { + assert(left.size() <= static_cast<size_t>(INT32_MAX)); + assert(right.size() <= static_cast<size_t>(INT32_MAX)); + threshold = std::min(threshold, static_cast<uint32_t>(std::numeric_limits<int32_t>::max())); uint32_t n = left.size(); uint32_t m = right.size(); - if (n > m) { - return calculate(right, left, threshold); - } - - // if one string is empty, the edit distance is necessarily the length - // of the other - if (n == 0) { - return m <= threshold ? std::optional(m) : std::nullopt; - } - if (m == 0) { - return n <= threshold ? std::optional(n) : std::nullopt; - } - - // the edit distance cannot be less than the length difference - if (m - n > threshold) { - return std::nullopt; + if (!prefix_match) { + // These optimizations are only valid when matching with target/source string symmetry. + // Correctness of the main matrix calculation loop should not depend on these. + if (n > m) { + return calculate(right, left, threshold, false); + } + // if one string is empty, the edit distance is necessarily the length + // of the other. + if (n == 0) { + return m <= threshold ? std::optional(m) : std::nullopt; + } + if (m == 0) { + return n <= threshold ? std::optional(n) : std::nullopt; + } + // the edit distance cannot be less than the length difference + if (m - n > threshold) { + return std::nullopt; + } + } else { + // A source (right) cannot be transformed into a target prefix (left) if doing + // so would require inserting more than max edits number of characters. + if ((n > m) && (n - m > threshold)) { + return std::nullopt; + } } std::vector<uint32_t> p(n+1); // 'previous' cost array, horizontally @@ -48,6 +63,7 @@ vespalib::LevenshteinDistance::calculate(std::span<const uint32_t> left, std::sp } // iterates through t + uint32_t min_edits = n; // prefix matching: worst-case to transform to target for (uint32_t j = 1; j <= m; ++j) { uint32_t rightJ = right[j - 1]; // jth character of right d[0] = j; @@ -56,9 +72,9 @@ vespalib::LevenshteinDistance::calculate(std::span<const uint32_t> left, std::sp uint32_t max = j > std::numeric_limits<uint32_t>::max() - threshold ? n : std::min(n, j + threshold); - // ignore entry left of leftmost if (min > 1) { + assert(static_cast<size_t>(min) <= d.size()); d[min - 1] = std::numeric_limits<uint32_t>::max(); } @@ -76,12 +92,35 @@ vespalib::LevenshteinDistance::calculate(std::span<const uint32_t> left, std::sp lowerBound = std::min(lowerBound, d[i]); } if (lowerBound > threshold) { - return std::nullopt; + if (!prefix_match) { + return std::nullopt; // Cannot match + } else { + break; // May already have matched via min_edits + } } std::swap(p, d); + // For prefix matching: + // By definition, the Levenshtein matrix cell at row `i`, column `j` + // provides the minimum number of edits required to transform a prefix of + // source string S (up to and including length `i`) into a prefix of target + // string T (up to and including length `j`). Since we want to match against + // the entire target (prefix query) string with length `n`, the problem is + // reduced to finding the minimum value of the `n`th column that is `<= k` + // (aggregated here and checked after the loop). + min_edits = std::min(p[n], min_edits); } - if (p[n] <= threshold) { + if (prefix_match) { + return ((min_edits <= threshold) ? std::optional<uint32_t>{min_edits} : std::nullopt); + } else if (p[n] <= threshold) { return {p[n]}; } return std::nullopt; } + +std::optional<uint32_t> +LevenshteinDistance::calculate(std::span<const uint32_t> left, std::span<const uint32_t> right, uint32_t threshold) +{ + return calculate(left, right, threshold, false); +} + +} // vespalib diff --git a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.h b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.h index f105dcdaaf4..e284e071731 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.h +++ b/vespalib/src/vespa/vespalib/fuzzy/levenshtein_distance.h @@ -19,7 +19,15 @@ namespace vespalib { */ class LevenshteinDistance { public: - static std::optional<uint32_t> calculate(std::span<const uint32_t> left, std::span<const uint32_t> right, uint32_t threshold); + // Iff `prefix_match` == true, `right` is the candidate to match against prefix `left` + static std::optional<uint32_t> calculate(std::span<const uint32_t> left, + std::span<const uint32_t> right, + uint32_t threshold, + bool prefix_match); + + static std::optional<uint32_t> calculate(std::span<const uint32_t> left, + std::span<const uint32_t> right, + uint32_t threshold); }; } diff --git a/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp b/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp index 388099a0358..e8b1bb5b415 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp +++ b/vespalib/src/vespa/vespalib/fuzzy/match_algorithm.hpp @@ -25,11 +25,14 @@ struct MatchAlgorithm { static constexpr uint8_t max_edits() noexcept { return MaxEdits; } /** - * Matches UTF-8 source string `source` against the target DFA, optionally generating + * Matches UTF-8/32 source string `source` against the target DFA, optionally generating * the successor string iff the source string is not within the maximum number of edits * of the target string. * - * The actual match loop is very simple: we try to match the DFA as far as we can + * The actual match loop is very simple, but with some subtle differences in semantics + * depending on whether the "full string" or "prefix" matching mode is used. + * + * For the full string matching mode, we try to match the DFA as far as we can * before either consuming all input (source string) characters or ending up in a non- * matching state before we have consumed all input. In the former case, we may be in * a matching state (consider matching "foo" with the target string "food"; after @@ -39,6 +42,15 @@ struct MatchAlgorithm { * If we end up in a matching state, all is well. We simply return a MatchResult with * the number of edits the state represents. * + * Prefix matching is very similar, but since we're interested in the number of edits + * required to transform a _prefix_ of the source into the target string, we cannot + * require matching the entire source string in the case that it is longer than the + * target (as that would not just be a prefix, after all). We consider a source prefix + * to be matched if _any_ DFA state is a match. This may be the initial state. + * Since the first match might not yield the minimum edit distance, we traverse the + * DFA until it no longer matches and return the smallest encountered distance as the + * final result. + * * The interesting bit happens the string does _not_ match and we are asked to provide a * _successor_ string that _does_ match and is strictly greater in lexicographic order. * @@ -143,6 +155,18 @@ struct MatchAlgorithm { * successor "foyd" (and _not_ "FOyd"), as the latter would imply a completely different * ordering when compared byte-wise against an implicitly lowercased dictionary. * + * The successor generation algorithm generalizes without changes to work as expected + * when prefix matching is used. This is because the target string represents the + * prefix and the successor consequently represents the smallest possible greater prefix. + * + * Proof by contradiction: S is a source string that does _not_ prefix match target + * string T, and S' is the returned successor. Assume that discarding (skipping) all + * candidate source strings C with S < C < S' misses at least one such candidate C' + * that has a prefix matching T. This means that it must be possible to traverse the + * DFA with source string C' and end up in a matching state. But by definition the + * successor S' is the smallest possible lexicographically greater string that can + * possibly match the DFA for T; a contradiction. + * * TODO let matcher know if source string is pre-normalized (i.e. lowercased). */ template <DfaMatcher Matcher, typename SuccessorT> @@ -158,6 +182,9 @@ struct MatchAlgorithm { StateType state = matcher.start(); while (u8_reader.hasMore()) { + if (matcher.is_prefix() && matcher.is_match(state)) { + return minimal_matching_prefix_distance(matcher, state, u8_reader); + } const auto pos_before_char = static_cast<uint32_t>(successor_out.size()); const uint32_t raw_mch = u8_reader.getChar(); const uint32_t mch = normalized_match_char(raw_mch, matcher.is_cased()); @@ -198,6 +225,9 @@ struct MatchAlgorithm { Utf8Reader u8_reader(source.data(), source.size()); StateType state = matcher.start(); while (u8_reader.hasMore()) { + if (matcher.is_prefix() && matcher.is_match(state)) { + return minimal_matching_prefix_distance(matcher, state, u8_reader); + } const uint32_t mch = normalized_match_char(u8_reader.getChar(), matcher.is_cased()); auto maybe_next = matcher.match_input(state, mch); if (matcher.can_match(maybe_next)) { @@ -214,6 +244,41 @@ struct MatchAlgorithm { } /** + * Follows the DFA (in an already matching state) until it no longer matches, keeping + * track of the minimum edit distance encountered. This is used to compute the minimum + * number of edits during prefix matching, as just returning the edit distance of the + * _first_ matching state may not result in the minimum. + * + * Example 1: matching the source string 'bananas' against the prefix target 'ban' with + * max edits 2 will be in a matching state already after consuming 'b' (can append 2 + * characters after), but had we processed the remaining characters we would end up with + * the actual prefix edit distance of 0. + * + * Example 2: matching the source string 'abcdef' against target 'acd' with max edits 2 + * will be in a matching state after 'a' (2 edits), but the minimal edit prefix is 'abcd' + * (1 edit). This demonstrates that we may have to process more than |target| number of + * source characters to get a minimal distance. + */ + template <DfaMatcher Matcher> + static MatchResult minimal_matching_prefix_distance(const Matcher& matcher, + typename Matcher::StateType state, + Utf8Reader& u8_reader) + { + auto min_edits = matcher.match_edit_distance(state); + while (u8_reader.hasMore()) { + const uint32_t mch = normalized_match_char(u8_reader.getChar(), matcher.is_cased()); + auto maybe_next = matcher.match_input(state, mch); + if (matcher.can_match(maybe_next)) { + state = maybe_next; + min_edits = std::min(min_edits, matcher.match_edit_distance(state)); + } else { + break; + } + } + return MatchResult::make_match(max_edits(), min_edits); + } + + /** * Instantly backtrack to the last possible branching point in the DFA where we can * choose some higher outgoing edge character value and still match the DFA. If the node * has a wildcard edge, we can bump the input char by one and generate the smallest diff --git a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h index 4ad203c9a46..a15f321f32d 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h +++ b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.h @@ -46,12 +46,13 @@ public: private: const std::vector<Lookup> _lookup; const bool _is_cased; + const bool _is_prefix; static std::vector<Lookup> make_lookup(const std::vector<uint32_t> &str); public: using MatchResult = LevenshteinDfa::MatchResult; - TableDfa(std::vector<uint32_t> str, bool is_cased); + TableDfa(std::vector<uint32_t> str, bool is_cased, bool is_prefix); ~TableDfa() override; [[nodiscard]] MatchResult match(std::string_view source) const override; [[nodiscard]] MatchResult match(std::string_view source, std::string& successor_out) const override; diff --git a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp index e7c721f644e..721cfe296d2 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp +++ b/vespalib/src/vespa/vespalib/fuzzy/table_dfa.hpp @@ -353,11 +353,13 @@ struct TableMatcher { const TableDfa<N>::Lookup *lookup; const uint32_t end; const bool cased; + const bool prefix; - TableMatcher(const TableDfa<N>::Lookup *lookup_in, uint32_t end_in, bool cased_in) - noexcept : lookup(lookup_in), end(end_in), cased(cased_in) {} + TableMatcher(const TableDfa<N>::Lookup *lookup_in, uint32_t end_in, bool cased_in, bool prefix_in) + noexcept : lookup(lookup_in), end(end_in), cased(cased_in), prefix(prefix_in) {} bool is_cased() const noexcept { return cased; } + bool is_prefix() const noexcept { return prefix; } static constexpr S start() noexcept { return S::start(); } uint8_t match_edit_distance(S s) const noexcept { return s.edits(end); } @@ -473,9 +475,10 @@ TableDfa<N>::make_lookup(const std::vector<uint32_t> &str)->std::vector<Lookup> } template <uint8_t N> -TableDfa<N>::TableDfa(std::vector<uint32_t> str, bool is_cased) +TableDfa<N>::TableDfa(std::vector<uint32_t> str, bool is_cased, bool is_prefix) : _lookup(make_lookup(str)), - _is_cased(is_cased) + _is_cased(is_cased), + _is_prefix(is_prefix) { } @@ -486,7 +489,7 @@ template <uint8_t N> LevenshteinDfa::MatchResult TableDfa<N>::match(std::string_view u8str) const { - TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased, _is_prefix); return MatchAlgorithm<N>::match(matcher, u8str); } @@ -494,7 +497,7 @@ template <uint8_t N> LevenshteinDfa::MatchResult TableDfa<N>::match(std::string_view u8str, std::string& successor_out) const { - TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased, _is_prefix); return MatchAlgorithm<N>::match(matcher, u8str, successor_out); } @@ -502,7 +505,7 @@ template <uint8_t N> LevenshteinDfa::MatchResult TableDfa<N>::match(std::string_view u8str, std::vector<uint32_t>& successor_out) const { - TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased); + TableMatcher<N> matcher(_lookup.data(), _lookup.size() - 1, _is_cased, _is_prefix); return MatchAlgorithm<N>::match(matcher, u8str, successor_out); } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index 72ef38a799e..ab02e7c1f05 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -1,12 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> +#include "nbo.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/bfloat16.h> -#include "nbo.h" +#include <vector> namespace vespalib { @@ -22,7 +22,7 @@ public: using Buffer = Array<char>; using Alloc = alloc::Alloc; enum State { ok=0, eof=0x01, oob=0x02}; - nbostream(size_t initialSize=1024); + explicit nbostream(size_t initialSize=1024); protected: nbostream(const void * buf, size_t sz, bool longLivedBuffer); public: @@ -220,7 +220,7 @@ public: class nbostream_longlivedbuf : public nbostream { public: - nbostream_longlivedbuf(size_t initialSize=1024); + explicit nbostream_longlivedbuf(size_t initialSize=1024); nbostream_longlivedbuf(const void * buf, size_t sz); }; diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index e9ae8a0ed58..394deb486aa 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -84,7 +84,7 @@ private: : _alloc(allocator->alloc(sz)), _allocator(allocator) { } - Alloc(const MemoryAllocator * allocator) noexcept + explicit Alloc(const MemoryAllocator * allocator) noexcept : _alloc(), _allocator(allocator) { } @@ -102,19 +102,19 @@ namespace vespalib { /// Rounds up to the closest number that is a power of 2 inline size_t -roundUp2inN(size_t minimum) { +roundUp2inN(size_t minimum) noexcept { return 2ul << Optimized::msbIdx(minimum - 1); } /// Rounds minElems up to the closest number where minElems*elemSize is a power of 2 inline size_t -roundUp2inN(size_t minElems, size_t elemSize) { +roundUp2inN(size_t minElems, size_t elemSize) noexcept { return roundUp2inN(minElems * elemSize)/elemSize; } template <typename T> size_t -roundUp2inN(size_t elems) { +roundUp2inN(size_t elems) noexcept { return roundUp2inN(elems, sizeof(T)); } diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index d0af62861f8..6cc22235361 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -83,7 +83,7 @@ public: using value_type = T; using size_type = size_t; - Array(const Alloc & initial=Alloc::alloc()); + Array(const Alloc & initial=Alloc::alloc()) noexcept; Array(size_t sz, const Alloc & initial=Alloc::alloc()); Array(Alloc && buf, size_t sz) noexcept; Array(Array &&rhs) noexcept; diff --git a/vespalib/src/vespa/vespalib/util/array.hpp b/vespalib/src/vespa/vespalib/util/array.hpp index 6ffbd63bd3a..33213df4cf6 100644 --- a/vespalib/src/vespa/vespalib/util/array.hpp +++ b/vespalib/src/vespa/vespalib/util/array.hpp @@ -148,15 +148,15 @@ void Array<T>::increase(size_t n) } template <typename T> -Array<T>::Array(const Alloc & initial) +Array<T>::Array(const Alloc & initial) noexcept : _array(initial.create(0)), _sz(0) { } template <typename T> -Array<T>::Array(Alloc && buf, size_t sz) noexcept : - _array(std::move(buf)), - _sz(sz) +Array<T>::Array(Alloc && buf, size_t sz) noexcept + : _array(std::move(buf)), + _sz(sz) { } @@ -170,25 +170,25 @@ Array<T>::Array(Array &&rhs) noexcept } template <typename T> -Array<T>::Array(size_t sz, const Alloc & initial) : - _array(initial.create(sz * sizeof(T))), - _sz(sz) +Array<T>::Array(size_t sz, const Alloc & initial) + : _array(initial.create(sz * sizeof(T))), + _sz(sz) { construct(array(0), _sz, std::is_trivially_default_constructible<T>()); } template <typename T> -Array<T>::Array(size_t sz, T value, const Alloc & initial) : - _array(initial.create(sz * sizeof(T))), - _sz(sz) +Array<T>::Array(size_t sz, T value, const Alloc & initial) + : _array(initial.create(sz * sizeof(T))), + _sz(sz) { construct(array(0), _sz, value, std::is_trivially_copyable<T>()); } template <typename T> -Array<T>::Array(const_iterator begin_, const_iterator end_, const Alloc & initial) : - _array(initial.create(begin_ != end_ ? sizeof(T) * (end_-begin_) : 0)), - _sz(end_-begin_) +Array<T>::Array(const_iterator begin_, const_iterator end_, const Alloc & initial) + : _array(initial.create(begin_ != end_ ? sizeof(T) * (end_-begin_) : 0)), + _sz(end_-begin_) { construct(array(0), begin_, _sz, std::is_trivially_copyable<T>()); } diff --git a/vespalib/src/vespa/vespalib/util/binary_hamming_distance.cpp b/vespalib/src/vespa/vespalib/util/binary_hamming_distance.cpp index 0e9393b7be4..5f63925bfef 100644 --- a/vespalib/src/vespa/vespalib/util/binary_hamming_distance.cpp +++ b/vespalib/src/vespa/vespalib/util/binary_hamming_distance.cpp @@ -4,25 +4,34 @@ namespace vespalib { -size_t binary_hamming_distance(const void *lhs, const void *rhs, size_t sz) { - uintptr_t addr_a = (uintptr_t) lhs; - uintptr_t addr_b = (uintptr_t) rhs; +namespace { + constexpr uint8_t WORD_SZ = sizeof (uint64_t); + constexpr uint8_t UNROLL_CNT = 4; + static_assert(sizeof(uint64_t) == 8); +} +size_t +binary_hamming_distance(const void *lhs, const void *rhs, size_t sz) noexcept { + auto addr_a = (uintptr_t) lhs; + auto addr_b = (uintptr_t) rhs; size_t sum = 0; size_t i = 0; - static_assert(sizeof(uint64_t) == 8); bool aligned = ((addr_a & 0x7) == 0) && ((addr_b & 0x7) == 0); if (__builtin_expect(aligned, true)) { - const uint64_t *words_a = static_cast<const uint64_t *>(lhs); - const uint64_t *words_b = static_cast<const uint64_t *>(rhs); - for (; i * 8 + 7 < sz; ++i) { - uint64_t xor_bits = words_a[i] ^ words_b[i]; - sum += __builtin_popcountl(xor_bits); + const auto *words_a = static_cast<const uint64_t *>(lhs); + const auto *words_b = static_cast<const uint64_t *>(rhs); + for (; (i+UNROLL_CNT) * WORD_SZ <= sz; i += UNROLL_CNT) { + for (uint8_t j=0; j < UNROLL_CNT; j++) { + sum += __builtin_popcountl(words_a[i+j] ^ words_b[i+j]); + } + } + for (; (i + 1) * WORD_SZ <= sz; ++i) { + sum += __builtin_popcountl(words_a[i] ^ words_b[i]); } } - if (__builtin_expect((i * 8 < sz), false)) { - const uint8_t *bytes_a = static_cast<const uint8_t *>(lhs); - const uint8_t *bytes_b = static_cast<const uint8_t *>(rhs); - for (i *= 8; i < sz; ++i) { + if (__builtin_expect((i * WORD_SZ < sz), false)) { + const auto *bytes_a = static_cast<const uint8_t *>(lhs); + const auto *bytes_b = static_cast<const uint8_t *>(rhs); + for (i *= WORD_SZ; i < sz; ++i) { uint64_t xor_bits = bytes_a[i] ^ bytes_b[i]; sum += __builtin_popcountl(xor_bits); } diff --git a/vespalib/src/vespa/vespalib/util/binary_hamming_distance.h b/vespalib/src/vespa/vespalib/util/binary_hamming_distance.h index 84bbbe71788..f5280903db1 100644 --- a/vespalib/src/vespa/vespalib/util/binary_hamming_distance.h +++ b/vespalib/src/vespa/vespalib/util/binary_hamming_distance.h @@ -10,5 +10,5 @@ namespace vespalib { * @param sz number of bytes in each blob * @return number of bits that differ when comparing the two blobs **/ -size_t binary_hamming_distance(const void *lhs, const void *rhs, size_t sz); +size_t binary_hamming_distance(const void *lhs, const void *rhs, size_t sz) noexcept; } diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.h b/vespalib/src/vespa/vespalib/util/generation_hold_list.h index 5d150c8a015..10ffb38816b 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.h +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.h @@ -22,11 +22,11 @@ private: struct ElemWithGen { T elem; generation_t gen; - ElemWithGen(T elem_in, generation_t gen_in) + ElemWithGen(T elem_in, generation_t gen_in) noexcept : elem(std::move(elem_in)), gen(gen_in) {} - size_t byte_size() const { + size_t byte_size() const noexcept { if constexpr (track_bytes_held) { return elem->byte_size(); } @@ -54,7 +54,7 @@ private: void reclaim_internal(generation_t oldest_used_gen, Func callback); public: - GenerationHoldList(); + GenerationHoldList() noexcept; ~GenerationHoldList(); /** @@ -99,7 +99,7 @@ public: template<typename Func> void reclaim_all(Func callback); - size_t get_held_bytes() const { return _held_bytes.load(std::memory_order_relaxed); } + size_t get_held_bytes() const noexcept { return _held_bytes.load(std::memory_order_relaxed); } // Static size of _phase_2_list might depend on std::deque implementation static constexpr size_t sizeof_phase_2_list = sizeof(ElemWithGenList); diff --git a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp index 532aa9abe28..dfe84f9cfc0 100644 --- a/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp +++ b/vespalib/src/vespa/vespalib/util/generation_hold_list.hpp @@ -12,7 +12,7 @@ void GenerationHoldList<T, track_bytes_held, use_deque>::assign_generation_internal(generation_t current_gen) { for (auto& elem : _phase_1_list) { - _phase_2_list.push_back(ElemWithGen(std::move(elem), current_gen)); + _phase_2_list.emplace_back(std::move(elem), current_gen); } _phase_1_list.clear(); } @@ -40,7 +40,7 @@ GenerationHoldList<T, track_bytes_held, use_deque>::reclaim_internal(generation_ } template <typename T, bool track_bytes_held, bool use_deque> -GenerationHoldList<T, track_bytes_held, use_deque>::GenerationHoldList() +GenerationHoldList<T, track_bytes_held, use_deque>::GenerationHoldList() noexcept : _phase_1_list(), _phase_2_list(), _held_bytes() diff --git a/vespalib/src/vespa/vespalib/util/generationholder.cpp b/vespalib/src/vespa/vespalib/util/generationholder.cpp index 9930cfedbe4..c172220bac2 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.cpp +++ b/vespalib/src/vespa/vespalib/util/generationholder.cpp @@ -12,7 +12,7 @@ template void GenerationHolderParent::reclaim_internal GenerationHeldBase::~GenerationHeldBase() = default; -GenerationHolder::GenerationHolder() +GenerationHolder::GenerationHolder() noexcept : GenerationHolderParent() { } diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index 00ce8182394..49ef038ad67 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -19,12 +19,12 @@ private: size_t _byte_size; public: - GenerationHeldBase(size_t byte_size_in) + GenerationHeldBase(size_t byte_size_in) noexcept : _byte_size(byte_size_in) { } virtual ~GenerationHeldBase(); - size_t byte_size() const { return _byte_size; } + constexpr size_t byte_size() const noexcept { return _byte_size; } }; using GenerationHolderParent = GenerationHoldList<GenerationHeldBase::UP, true, false>; @@ -35,7 +35,7 @@ using GenerationHolderParent = GenerationHoldList<GenerationHeldBase::UP, true, */ class GenerationHolder : public GenerationHolderParent { public: - GenerationHolder(); + GenerationHolder() noexcept; }; } diff --git a/vespalib/src/vespa/vespalib/util/optimized.h b/vespalib/src/vespa/vespalib/util/optimized.h index 12d1b770904..f651a411367 100644 --- a/vespalib/src/vespa/vespalib/util/optimized.h +++ b/vespalib/src/vespa/vespalib/util/optimized.h @@ -16,15 +16,15 @@ namespace vespalib { class Optimized { public: - static int msbIdx(unsigned int v); - static int msbIdx(unsigned long v); - static int msbIdx(unsigned long long v); - static int lsbIdx(unsigned int v); - static int lsbIdx(unsigned long v); - static int lsbIdx(unsigned long long v); - static constexpr int popCount(unsigned int v) { return __builtin_popcount(v); } - static constexpr int popCount(unsigned long v) { return __builtin_popcountl(v); } - static constexpr int popCount(unsigned long long v) { return __builtin_popcountll(v); } + static int msbIdx(unsigned int v) noexcept; + static int msbIdx(unsigned long v) noexcept; + static int msbIdx(unsigned long long v) noexcept; + static int lsbIdx(unsigned int v) noexcept; + static int lsbIdx(unsigned long v) noexcept; + static int lsbIdx(unsigned long long v) noexcept; + static constexpr int popCount(unsigned int v) noexcept { return __builtin_popcount(v); } + static constexpr int popCount(unsigned long v) noexcept { return __builtin_popcountl(v); } + static constexpr int popCount(unsigned long long v) noexcept { return __builtin_popcountll(v); } }; /** @@ -64,43 +64,43 @@ public: **/ #ifdef __x86_64__ -inline int Optimized::msbIdx(unsigned int v) { +inline int Optimized::msbIdx(unsigned int v) noexcept { unsigned int result; __asm __volatile("bsrl %0,%0" : "=r" (result) : "0" (v)); return result; } -inline int Optimized::lsbIdx(unsigned int v) { +inline int Optimized::lsbIdx(unsigned int v) noexcept { unsigned int result; __asm __volatile("bsfl %0,%0" : "=r" (result) : "0" (v)); return result; } -inline int Optimized::msbIdx(unsigned long v) { +inline int Optimized::msbIdx(unsigned long v) noexcept { unsigned long result; __asm __volatile("bsrq %0,%0" : "=r" (result) : "0" (v)); return result; } -inline int Optimized::lsbIdx(unsigned long v) { +inline int Optimized::lsbIdx(unsigned long v) noexcept { unsigned long result; __asm __volatile("bsfq %0,%0" : "=r" (result) : "0" (v)); return result; } -inline int Optimized::msbIdx(unsigned long long v) { +inline int Optimized::msbIdx(unsigned long long v) noexcept { unsigned long long result; __asm __volatile("bsrq %0,%0" : "=r" (result) : "0" (v)); return result; } -inline int Optimized::lsbIdx(unsigned long long v) { +inline int Optimized::lsbIdx(unsigned long long v) noexcept { unsigned long long result; __asm __volatile("bsfq %0,%0" : "=r" (result) : "0" (v)); return result; } #else -inline int Optimized::msbIdx(unsigned int v) { return v ? sizeof(unsigned int) * 8 - 1 - __builtin_clz(v) : 0; } -inline int Optimized::msbIdx(unsigned long v) { return v ? sizeof(unsigned long) * 8 - 1 - __builtin_clzl(v) : 0; } -inline int Optimized::msbIdx(unsigned long long v) { return v ? sizeof(unsigned long long) * 8 - 1 - __builtin_clzll(v) : 0; } -inline int Optimized::lsbIdx(unsigned int v) { return v ? __builtin_ctz(v) : 0; } -inline int Optimized::lsbIdx(unsigned long v) { return v ? __builtin_ctzl(v) : 0; } -inline int Optimized::lsbIdx(unsigned long long v) { return v ? __builtin_ctzll(v) : 0; } +inline int Optimized::msbIdx(unsigned int v) noexcept { return v ? sizeof(unsigned int) * 8 - 1 - __builtin_clz(v) : 0; } +inline int Optimized::msbIdx(unsigned long v) noexcept { return v ? sizeof(unsigned long) * 8 - 1 - __builtin_clzl(v) : 0; } +inline int Optimized::msbIdx(unsigned long long v) noexcept { return v ? sizeof(unsigned long long) * 8 - 1 - __builtin_clzll(v) : 0; } +inline int Optimized::lsbIdx(unsigned int v) noexcept { return v ? __builtin_ctz(v) : 0; } +inline int Optimized::lsbIdx(unsigned long v) noexcept { return v ? __builtin_ctzl(v) : 0; } +inline int Optimized::lsbIdx(unsigned long long v) noexcept { return v ? __builtin_ctzll(v) : 0; } #endif #define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/vespamalloc/src/vespamalloc/util/callstack.cpp b/vespamalloc/src/vespamalloc/util/callstack.cpp index a0645f06815..b8449c89a72 100644 --- a/vespamalloc/src/vespamalloc/util/callstack.cpp +++ b/vespamalloc/src/vespamalloc/util/callstack.cpp @@ -53,6 +53,7 @@ const void * StackEntry::_stopAddr = nullptr; size_t StackEntry::fillStack(StackEntry *stack, size_t nelems) { + // GNU extension: Variable-length automatic array void * retAddr[nelems]; int sz = backtrace(retAddr, nelems); if ((sz > 0) && (size_t(sz) <= nelems)) { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MemoryFileSystem.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MemoryFileSystem.java index c4cc1541ca6..d05c8f11ad4 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MemoryFileSystem.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MemoryFileSystem.java @@ -53,7 +53,7 @@ class MemoryFileSystem extends FileSystem { @Override public Iterable<Path> getRootDirectories() { - return Collections.singleton(Paths.get("/")); + return Set.of(Paths.get("/")); } @Override @@ -63,7 +63,7 @@ class MemoryFileSystem extends FileSystem { @Override public Set<String> supportedFileAttributeViews() { - return Collections.emptySet(); + return Set.of(); } @Override diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/CuratorOperations.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/CuratorOperations.java index 6a3184fa3e1..ed114c7c340 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/CuratorOperations.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/CuratorOperations.java @@ -5,7 +5,6 @@ import com.yahoo.path.Path; import com.yahoo.vespa.curator.Curator; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -62,7 +61,7 @@ public class CuratorOperations { * This does not fail, but returns an empty list if the path does not exist. */ public static List<CuratorOperation> deleteAll(String path, Curator curator) { - if ( ! curator.exists(Path.fromString(path))) return Collections.emptyList(); + if ( ! curator.exists(Path.fromString(path))) return List.of(); List<CuratorOperation> operations = new ArrayList<>(); deleteRecursively(Path.fromString(path), operations, curator); diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java index 83cfaf11a92..f6dfb0fa4d9 100644 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java +++ b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/common/ClientX509Util.java @@ -23,7 +23,6 @@ import io.netty.handler.ssl.DelegatingSslContext; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslProvider; -import java.util.Arrays; import javax.net.ssl.KeyManager; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; @@ -32,6 +31,8 @@ import javax.net.ssl.TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * X509 utilities specific for client-server communication framework. * <p> @@ -196,9 +197,9 @@ public class ClientX509Util extends X509Util { if (getSslProvider(config) != SslProvider.JDK) { return null; } - return Arrays.asList(X509Util.getDefaultCipherSuites()); + return List.of(X509Util.getDefaultCipherSuites()); } else { - return Arrays.asList(cipherSuitesInput.split(",")); + return List.of(cipherSuitesInput.split(",")); } } diff --git a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java index 8ea94fd4daf..99c4ae16dce 100644 --- a/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.9.2/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java @@ -19,7 +19,6 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; -import java.util.Collections; import java.util.Map; import org.apache.zookeeper.jmx.MBeanRegistry; import org.apache.zookeeper.server.DataTreeBean; @@ -61,7 +60,7 @@ public abstract class LearnerZooKeeperServer extends QuorumZooKeeperServer { if (sessionTracker != null) { return ((LearnerSessionTracker) sessionTracker).snapshot(); } - Map<Long, Integer> map = Collections.emptyMap(); + Map<Long, Integer> map = Map.of(); return map; } diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java index 83cfaf11a92..f6dfb0fa4d9 100644 --- a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java @@ -23,7 +23,6 @@ import io.netty.handler.ssl.DelegatingSslContext; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslProvider; -import java.util.Arrays; import javax.net.ssl.KeyManager; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; @@ -32,6 +31,8 @@ import javax.net.ssl.TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * X509 utilities specific for client-server communication framework. * <p> @@ -196,9 +197,9 @@ public class ClientX509Util extends X509Util { if (getSslProvider(config) != SslProvider.JDK) { return null; } - return Arrays.asList(X509Util.getDefaultCipherSuites()); + return List.of(X509Util.getDefaultCipherSuites()); } else { - return Arrays.asList(cipherSuitesInput.split(",")); + return List.of(cipherSuitesInput.split(",")); } } diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java index 8ea94fd4daf..99c4ae16dce 100644 --- a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LearnerZooKeeperServer.java @@ -19,7 +19,6 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; -import java.util.Collections; import java.util.Map; import org.apache.zookeeper.jmx.MBeanRegistry; import org.apache.zookeeper.server.DataTreeBean; @@ -61,7 +60,7 @@ public abstract class LearnerZooKeeperServer extends QuorumZooKeeperServer { if (sessionTracker != null) { return ((LearnerSessionTracker) sessionTracker).snapshot(); } - Map<Long, Integer> map = Collections.emptyMap(); + Map<Long, Integer> map = Map.of(); return map; } |