diff options
143 files changed, 1898 insertions, 1652 deletions
diff --git a/application/pom.xml b/application/pom.xml index 61cea1a1826..af25bda0f07 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -97,17 +97,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - <exclusions> - <exclusion> - <!-- We want to get this via jdisc-core --> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.antlr</groupId> <artifactId>antlr-runtime</artifactId> </dependency> diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java index dbfac4e730f..faa33542f5c 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java @@ -18,10 +18,8 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; import java.util.zip.ZipEntry; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -35,7 +33,7 @@ public class BundleTest { // If bundle-plugin-test is compiled in a mvn command that also built dependencies, e.g. jrt, // the artifact is jrt.jar, otherwise the installed and versioned artifact // is used: jrt-7-SNAPSHOT.jar or e.g. jrt-7.123.45.jar. - private static String snapshotOrVersionOrNone = "(-\\d+((-SNAPSHOT)|((\\.\\d+(\\.\\d+)?)?))?)?\\.jar"; + private static final String snapshotOrVersionOrNone = "(-\\d+((-SNAPSHOT)|((\\.\\d+(\\.\\d+)?)?))?)?\\.jar"; private JarFile jarFile; private Attributes mainAttributes; @@ -67,12 +65,12 @@ public class BundleTest { // Because of snapshot builds, we can only verify the major version. int majorBundleVersion = Integer.valueOf(bundleVersion.substring(0, bundleVersion.indexOf('.'))); - assertThat(majorBundleVersion, is(VespaVersion.major)); + assertEquals(VespaVersion.major, majorBundleVersion); } @Test public void require_that_bundle_symbolic_name_matches_pom_artifactId() { - assertThat(mainAttributes.getValue("Bundle-SymbolicName"), is("main")); + assertEquals("main", mainAttributes.getValue("Bundle-SymbolicName")); } @Test @@ -80,29 +78,29 @@ public class BundleTest { String importPackage = mainAttributes.getValue("Import-Package"); // From SimpleSearcher - assertThat(importPackage, containsString("com.yahoo.prelude.hitfield")); + assertTrue(importPackage.contains("com.yahoo.prelude.hitfield")); // From SimpleSearcher2 - assertThat(importPackage, containsString("com.yahoo.processing")); - assertThat(importPackage, containsString("com.yahoo.metrics.simple")); - assertThat(importPackage, containsString("com.google.inject")); + assertTrue(importPackage.contains("com.yahoo.processing")); + assertTrue(importPackage.contains("com.yahoo.metrics.simple")); + assertTrue(importPackage.contains("com.google.inject")); } @Test public void require_that_manifest_contains_manual_imports() { String importPackage = mainAttributes.getValue("Import-Package"); - assertThat(importPackage, containsString("manualImport.withoutVersion")); - assertThat(importPackage, containsString("manualImport.withVersion;version=\"12.3.4\"")); + assertTrue(importPackage.contains("manualImport.withoutVersion")); + assertTrue(importPackage.contains("manualImport.withVersion;version=\"12.3.4\"")); for (int i=1; i<=2; ++i) - assertThat(importPackage, containsString("multiple.packages.with.the.same.version" + i + ";version=\"[1,2)\"")); + assertTrue(importPackage.contains("multiple.packages.with.the.same.version" + i + ";version=\"[1,2)\"")); } @Test public void require_that_manifest_contains_exports() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("com.yahoo.test;version=1.2.3.RELEASE")); + assertTrue(exportPackage.contains("com.yahoo.test;version=1.2.3.RELEASE")); } @Test @@ -110,7 +108,7 @@ public class BundleTest { // generated bundle. (It's compile scoped in pom.xml to be added to the bundle-cp.) public void require_that_manifest_contains_bundle_class_path() { String bundleClassPath = mainAttributes.getValue("Bundle-ClassPath"); - assertThat(bundleClassPath, containsString(".,")); + assertTrue(bundleClassPath.contains(".,")); Pattern jrtPattern = Pattern.compile("dependencies/jrt" + snapshotOrVersionOrNone); assertTrue("Bundle class path did not contain jrt.", jrtPattern.matcher(bundleClassPath).find()); @@ -139,7 +137,7 @@ public class BundleTest { @Test public void require_that_web_inf_url_is_propagated_to_the_manifest() { String webInfUrl = mainAttributes.getValue("WebInfUrl"); - assertThat(webInfUrl, containsString("/WEB-INF/web.xml")); + assertTrue(webInfUrl.contains("/WEB-INF/web.xml")); } } diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java index 36d3faba81f..f99f1583324 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/ExportPackageVersionTest.java @@ -10,9 +10,8 @@ import java.util.jar.Attributes; import java.util.jar.JarFile; import static com.yahoo.container.plugin.BundleTest.findBundleJar; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Verifies that the 'useArtifactVersionForExportPackages' setting for the bundle-plugin works as intended. @@ -42,16 +41,16 @@ public class ExportPackageVersionTest { String expectedExport = "ai.vespa.noversion;version=" + bundleVersion; String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString(expectedExport)); + assertTrue(exportPackage.contains(expectedExport)); // Verify that there is no qualifier - assertThat(exportPackage, not(containsString(expectedExport + "."))); + assertFalse(exportPackage.contains(expectedExport + ".")); } @Test public void explicit_version_in_ExportPackage_annotation_overrides_artifact_version() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("ai.vespa.explicitversion;version=2.4.6.RELEASE")); + assertTrue(exportPackage.contains("ai.vespa.explicitversion;version=2.4.6.RELEASE")); } @Test @@ -60,13 +59,13 @@ public class ExportPackageVersionTest { // TODO: This test should have checked for a fixed version of the dependency bundle, different than the main bundle version. // See comment in the dependency bundle's pom.xml for why this is not the case. - assertThat(exportPackage, containsString("ai.vespa.noversion_dep;version=" + bundleVersion)); + assertTrue(exportPackage.contains("ai.vespa.noversion_dep;version=" + bundleVersion)); } @Test public void explicit_version_in_ExportPackage_annotation_overrides_artifact_version_of_compile_scoped_dependency() { String exportPackage = mainAttributes.getValue("Export-Package"); - assertThat(exportPackage, containsString("ai.vespa.explicitversion_dep;version=3.6.9.RELEASE")); + assertTrue(exportPackage.contains("ai.vespa.explicitversion_dep;version=3.6.9.RELEASE")); } } diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 07811d23581..9ed4c47c023 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -39,15 +39,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </dependency> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java index 2e44e69bac6..2b5941cc5aa 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/bundle/AnalyzeBundle.java @@ -13,7 +13,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.jar.Manifest; -import java.util.stream.Collectors; /** * Static utilities for analyzing jar files. diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java index 3b2aa2369ef..04fa18c9bc5 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ExportPackages.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -58,6 +59,19 @@ public class ExportPackages { public String getValue() { return value; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Parameter parameter = (Parameter) o; + return Objects.equals(name, parameter.name) && Objects.equals(value, parameter.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } } public static Set<String> packageNames(Collection<Export> exports) { diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java index 8fba5983049..62ecd780d7e 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/bundle/AnalyzeBundleTest.java @@ -3,25 +3,17 @@ package com.yahoo.container.plugin.bundle; import com.yahoo.container.plugin.osgi.ExportPackages; import com.yahoo.container.plugin.osgi.ExportPackages.Export; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.File; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.Matchers.startsWith; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author Tony Vaagenes @@ -36,7 +28,7 @@ public class AnalyzeBundleTest { public AnalyzeBundleTest() { File notOsgi = new File(jarDir, "notAOsgiBundle.jar"); File simple = new File(jarDir, "simple1.jar"); - exports = AnalyzeBundle.exportedPackagesAggregated(Arrays.asList(notOsgi, simple)); + exports = AnalyzeBundle.exportedPackagesAggregated(List.of(notOsgi, simple)); exportsByPackageName = ExportPackages.exportsByPackageName(this.exports); } @@ -46,45 +38,28 @@ public class AnalyzeBundleTest { @Test public void require_that_non_osgi_bundles_are_ignored() { - assertThat(exportsByPackageName.keySet(), not(hasItem("com.yahoo.sample.exported.package.ignored"))); + assertFalse(exportsByPackageName.containsKey("com.yahoo.sample.exported.package.ignored")); } @Test public void require_that_exports_are_retrieved_from_manifest_in_jars() { - assertThat(exportsByPackageName.keySet().size(), is(1)); - assertThat(exportsByPackageName.keySet(), hasItem("com.yahoo.sample.exported.package")); + assertEquals(1, exportsByPackageName.keySet().size()); + assertTrue(exportsByPackageName.containsKey("com.yahoo.sample.exported.package")); } @Test public void exported_class_names_can_be_retrieved() { - assertThat(ExportPackages.packageNames(exports), is(new HashSet<>(exports.get(0).getPackageNames()))); + assertEquals(ExportPackages.packageNames(exports), exports.get(0).getPackageNames().stream().collect(Collectors.toSet())); } - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void require_that_invalid_exports_throws_exception() { - exception.expect(Exception.class); - - exception.expectMessage(containsString("Invalid manifest in bundle")); - exception.expectMessage(matchesPattern("Invalid manifest in bundle '.*errorExport.jar'")); - exception.expectCause(throwableMessage(startsWith("Failed parsing Export-Package"))); - - AnalyzeBundle.exportedPackages(jarFile("errorExport.jar")); - } - - private TypeSafeMatcher<String> matchesPattern(String pattern) { - return new TypeSafeMatcher<String>() { - @Override - protected boolean matchesSafely(String s) { - return s.matches(pattern); - } - - @Override - public void describeTo(Description description) { - description.appendText("expects String that matches the pattern ").appendValue(pattern); - } - }; + try { + AnalyzeBundle.exportedPackages(jarFile("errorExport.jar")); + fail(); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("Invalid manifest in bundle 'src/test/resources/jar/errorExport.jar'")); + assertTrue(e.getCause().getMessage(), e.getCause().getMessage().startsWith("Failed parsing Export-Package")); + } } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java index 44ef26646f0..518f65ded17 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -12,28 +12,22 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodInvocation; import com.yahoo.osgi.annotation.ExportPackage; import com.yahoo.osgi.annotation.Version; -import org.hamcrest.Matcher; -import org.hamcrest.Matchers; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import javax.security.auth.login.LoginException; import java.awt.Image; import java.awt.image.ImagingOpException; import java.awt.image.Kernel; +import java.util.List; import java.util.Optional; import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; import static com.yahoo.container.plugin.classanalysis.TestUtilities.classFile; import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; -import static com.yahoo.container.plugin.classanalysis.TestUtilities.throwableMessage; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests that analysis of class files works. @@ -45,128 +39,127 @@ public class AnalyzeClassTest { @Test public void require_that_full_class_name_is_returned() { - assertThat(analyzeClass(Base.class).getName(), is(name(Base.class))); + assertEquals(name(Base.class), analyzeClass(Base.class).getName()); } @Test public void require_that_base_class_is_included() { - assertThat(analyzeClass(Derived.class).getReferencedClasses(), hasItem(name(Base.class))); + assertTrue(analyzeClass(Derived.class).getReferencedClasses().contains(name(Base.class))); } @Test public void require_that_implemented_interfaces_are_included() { - assertThat(analyzeClass(Base.class).getReferencedClasses(), - allOf(hasItem(name(Interface1.class)), hasItem(name(Interface2.class)))); + assertTrue(analyzeClass(Base.class).getReferencedClasses().containsAll( + List.of(name(Interface1.class), name(Interface2.class)))); } @Test public void require_that_interface_can_be_analyzed() { ClassFileMetaData classMetaData = analyzeClass(Interface1.class); - assertThat(classMetaData.getName(), is(name(Interface1.class))); - assertThat(classMetaData.getReferencedClasses(), hasItem(name(Interface2.class))); + assertEquals(name(Interface1.class), classMetaData.getName()); + assertTrue(classMetaData.getReferencedClasses().contains(name(Interface2.class))); } @Test public void require_that_return_type_is_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Image.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Image.class))); } @Test public void require_that_parameters_are_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(Kernel.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Kernel.class))); } @Test public void require_that_exceptions_are_included() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), hasItem(name(ImagingOpException.class))); + assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(ImagingOpException.class))); } @Test public void require_that_basic_types_ignored() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), - not(Matchers.<Iterable<? super String>>anyOf(hasItem("int"), hasItem("float")))); + List.of("int", "float").forEach(type -> + assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test public void require_that_arrays_of_basic_types_ignored() { - assertThat(analyzeClass(Interface1.class).getReferencedClasses(), - not(Matchers.<Iterable<? super String>>anyOf(hasItem("int[]"), hasItem("int[][]")))); + List.of("int[]", "int[][]").forEach(type -> + assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test public void require_that_instance_field_types_are_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(String.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(String.class))); } @Test public void require_that_static_field_types_are_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(java.util.List.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(java.util.List.class))); } @Test public void require_that_field_annotation_is_included() { - assertThat(analyzeClass(Fields.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_class_annotation_is_included() { - assertThat(analyzeClass(ClassAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(ClassAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_method_annotation_is_included() { - assertThat(analyzeClass(MethodAnnotation.class).getReferencedClasses(), hasItem(name(DummyAnnotation.class))); + assertTrue(analyzeClass(MethodAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test public void require_that_export_package_annotations_are_ignored() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) - .getReferencedClasses(), not(Matchers.<Iterable<? super String>>anyOf( - hasItem(name(ExportPackage.class)), hasItem(name(Version.class))))); + List.of(ExportPackage.class, Version.class).forEach(type -> + assertFalse(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) + .getReferencedClasses().contains(type))); } @Test public void require_that_export_annotations_are_processed() { - assertThat( - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage(), - is(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")))); + assertEquals(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")), + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage()); } - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void require_that_export_annotations_are_validated() { - expectedException.expect(RuntimeException.class); - expectedException.expectMessage(containsString("invalid/package-info")); - expectedException.expectCause(throwableMessage(containsString("qualifier must follow the format"))); - expectedException.expectCause(throwableMessage(containsString("'EXAMPLE INVALID QUALIFIER'"))); - Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); + try { + Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); + fail(); + } catch (RuntimeException e) { + assertTrue(e.getMessage().contains("invalid/package-info")); + assertTrue(e.getCause().getMessage().contains("qualifier must follow the format")); + assertTrue(e.getCause().getMessage().contains("'EXAMPLE INVALID QUALIFIER'")); + } } @Test public void require_that_catch_clauses_are_included() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) - .getReferencedClasses(), hasItem(name(LoginException.class))); + assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) + .getReferencedClasses().contains(name(LoginException.class))); } @Test public void require_that_class_references_are_included() { - assertThat(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) - .getReferencedClasses(), hasItem(name(Interface1.class))); + assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) + .getReferencedClasses().contains(name(Interface1.class))); } @Test public void require_that_return_type_of_method_invocations_are_included() { - assertThat(analyzeClass(MethodInvocation.class).getReferencedClasses(), hasItem(name(Image.class))); + assertTrue(analyzeClass(MethodInvocation.class).getReferencedClasses().contains(name(Image.class))); } @Test public void require_that_attributes_are_included() { //Uses com/coremedia/iso/Utf8.class from com.googlecode.mp4parser:isoparser:1.0-RC-1 - assertThat(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses(), - hasItem("org.aspectj.weaver.MethodDeclarationLineNumber")); + assertTrue(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses() + .contains("org.aspectj.weaver.MethodDeclarationLineNumber")); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java index b7565a4dad8..4502f88d158 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -11,13 +11,12 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.Methods; import org.junit.Test; import java.io.PrintStream; +import java.util.List; import static com.yahoo.container.plugin.classanalysis.TestUtilities.analyzeClass; import static com.yahoo.container.plugin.classanalysis.TestUtilities.name; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests that classes used in method bodies are included in the imports list. @@ -27,47 +26,46 @@ import static org.junit.Assert.assertThat; public class AnalyzeMethodBodyTest { @Test public void require_that_class_of_locals_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Base.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Base.class))); } @Test public void require_that_class_of_locals_in_static_method_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Derived.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Derived.class))); } @Test public void require_that_field_references_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), - allOf(hasItem(name(java.util.List.class)), hasItem(name(Fields.class)))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().containsAll(List.of(name(java.util.List.class), name(Fields.class)))); } @Test public void require_that_class_owning_field_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(System.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(System.class))); } @Test public void require_that_class_containing_method_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(PrintStream.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(PrintStream.class))); } @Test public void require_that_element_of_new_multidimensional_array_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Interface1.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface1.class))); } @Test public void require_that_basic_arrays_are_not_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), not(hasItem("int[]"))); + assertFalse(analyzeClass(Methods.class).getReferencedClasses().contains("int[]")); } @Test public void require_that_container_generic_parameters_are_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(Dummy.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Dummy.class))); } @Test public void require_that_class_owning_method_handler_is_included() { - assertThat(analyzeClass(Methods.class).getReferencedClasses(), hasItem(name(ClassWithMethod.class))); + assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(ClassWithMethod.class))); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java index cfee980cb91..5bef1457c5f 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java @@ -3,11 +3,10 @@ package com.yahoo.container.plugin.classanalysis; import org.junit.Test; +import java.util.Map; import java.util.Set; -import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -16,9 +15,9 @@ public class PackageTallyTest { @Test public void referenced_packages_missing_from_are_detected() { - PackageTally tally = new PackageTally(emptyMap(), Set.of("p1", "java.util", "com.yahoo.api.annotations", "missing")); + PackageTally tally = new PackageTally(Map.of(), Set.of("p1", "java.util", "com.yahoo.api.annotations", "missing")); Set<String> missingPackages = tally.referencedPackagesMissingFrom(Set.of("p1")); - assertThat(missingPackages, is(Set.of("missing"))); + assertEquals(Set.of("missing"), missingPackages); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java index f0784cf80bc..9eacc0625b5 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/TestUtilities.java @@ -1,10 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.classanalysis; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; - import java.io.File; /** @@ -23,18 +19,4 @@ public class TestUtilities { public static String name(Class<?> clazz) { return clazz.getName(); } - - public static TypeSafeMatcher<Throwable> throwableMessage(final Matcher<String> matcher) { - return new TypeSafeMatcher<Throwable>() { - @Override - protected boolean matchesSafely(Throwable throwable) { - return matcher.matches(throwable.getMessage()); - } - - @Override - public void describeTo(Description description) { - description.appendText("expects Throwable and a message matching ").appendDescriptionOf(matcher); - } - }; - } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java index cb976e46655..2c9b32a325f 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ExportPackageParserTest.java @@ -3,17 +3,13 @@ package com.yahoo.container.plugin.osgi; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Ignore; import org.junit.Test; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * @author Tony Vaagenes @@ -26,41 +22,41 @@ public class ExportPackageParserTest { public void require_that_package_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("sample.exported.package"); - assertThat(exports.size(), is(1)); - assertThat(exports.get(0).getParameters(), empty()); - assertThat(exports.get(0).getPackageNames(), contains("sample.exported.package")); + assertEquals(1, exports.size()); + assertTrue(exports.get(0).getParameters().isEmpty()); + assertTrue(exports.get(0).getPackageNames().contains("sample.exported.package")); } @Test public void require_that_version_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("com.yahoo.sample.exported.package;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("com.yahoo.sample.exported.package")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("com.yahoo.sample.exported.package")); + assertTrue(export.getParameters().contains(versionParameter)); } @Test public void require_that_multiple_packages_with_same_parameters_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package1;exported.package2;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1", "exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().containsAll(List.of("exported.package1", "exported.package2"))); + assertTrue(export.getParameters().contains(versionParameter)); } @Test public void require_that_spaces_between_separators_are_allowed() { List<Export> exports = ExportPackageParser.parseExports("exported.package1 , exported.package2 ; version = \"1.2.3.sample\" "); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); + assertTrue(export.getPackageNames().contains("exported.package1")); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().contains(versionParameter)); } @SuppressWarnings("unchecked") @@ -68,39 +64,39 @@ public class ExportPackageParserTest { public void require_that_multiple_parameters_for_a_package_is_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package;version=\"1.2.3.sample\";param2=true"); - assertThat(exports.size(), is(1)); + assertEquals(1, exports.size()); Export export = exports.get(0); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter), parameterMatching("param2", "true"))); + assertTrue(export.getParameters().containsAll(List.of(versionParameter, new Parameter("param2", "true")))); } @Test public void require_that_multiple_exports_are_parsed_correctly() { List<Export> exports = ExportPackageParser.parseExports("exported.package1,exported.package2"); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); Export export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().isEmpty()); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().isEmpty()); exports = ExportPackageParser.parseExports("exported.package1;version=\"1.2.3.sample\",exported.package2"); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().contains(versionParameter)); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().isEmpty()); exports = ExportPackageParser.parseExports("exported.package1,exported.package2;version=\"1.2.3.sample\""); - assertThat(exports.size(), is(2)); + assertEquals(2, exports.size()); export = exports.get(0); - assertThat(export.getPackageNames(), contains("exported.package1")); - assertThat(export.getParameters(), empty()); + assertTrue(export.getPackageNames().contains("exported.package1")); + assertTrue(export.getParameters().isEmpty()); export = exports.get(1); - assertThat(export.getPackageNames(), contains("exported.package2")); - assertThat(export.getParameters(), contains(parameterMatching(versionParameter))); + assertTrue(export.getPackageNames().contains("exported.package2")); + assertTrue(export.getParameters().contains(versionParameter)); } // TODO: MAVEN_OPTS are not propagated by the maven-surefire-plugin. Either try to fix the underlying problem or set -Xss in plugin config. @@ -278,21 +274,4 @@ public class ExportPackageParserTest { ExportPackageParser.parseExports(exportHeader); } - private static TypeSafeMatcher<Parameter> parameterMatching(final String name, final String value) { - return new TypeSafeMatcher<Parameter>() { - @Override - protected boolean matchesSafely(Parameter parameter) { - return parameter.getName().equals(name) && parameter.getValue().equals(value); - } - - @Override - public void describeTo(Description description) { - description.appendText("Parameter with name ").appendValue(name).appendText(" with value ").appendValue(value); - } - }; - } - - private static TypeSafeMatcher<Parameter> parameterMatching(final Parameter param) { - return parameterMatching(param.getName(), param.getValue()); - } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java index d6b5b0fe9e0..ee7b6c9c2f1 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/osgi/ImportPackageTest.java @@ -4,25 +4,19 @@ package com.yahoo.container.plugin.osgi; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ExportPackages.Parameter; import com.yahoo.container.plugin.osgi.ImportPackages.Import; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Tony Vaagenes @@ -30,58 +24,56 @@ import static org.junit.Assert.assertThat; */ public class ImportPackageTest { private static final String PACKAGE_NAME = "com.yahoo.exported"; - private Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); - private Map<String, Export> exports = exportByPackageName(new Export(singletonList(PACKAGE_NAME), Collections.emptyList())); - private Map<String, Export> exportsWithVersion = exportByPackageName( - new Export(singletonList(PACKAGE_NAME), singletonList(new Parameter("version", "1.3")))); + private final Set<String> referencedPackages = Collections.singleton(PACKAGE_NAME); + private final Map<String, Export> exports = exportByPackageName(new Export(List.of(PACKAGE_NAME), Collections.emptyList())); + private final Map<String, Export> exportsWithVersion = exportByPackageName( + new Export(List.of(PACKAGE_NAME), List.of(new Parameter("version", "1.3")))); private static Map<String, Export> exportByPackageName(Export export) { - return ExportPackages.exportsByPackageName(singletonList(export)); + return ExportPackages.exportsByPackageName(List.of(export)); } @Test public void require_that_non_implemented_import_with_matching_export_is_included() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), exports); - assertThat(imports, contains(importMatching(PACKAGE_NAME, ""))); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), exports); + assertEquals(1, imports.stream().filter(imp -> imp.packageName().equals(PACKAGE_NAME) && imp.version().isEmpty()).count()); } @Test public void require_that_non_implemented_import_without_matching_export_is_excluded() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), emptyMap()); - assertThat(imports, empty()); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), Map.of()); + assertTrue(imports.isEmpty()); } @Test public void require_that_implemented_import_with_matching_export_is_excluded() { Set<Import> imports = calculateImports(referencedPackages, referencedPackages, exports); - - assertThat(imports, empty()); + assertTrue(imports.isEmpty()); } @Test public void require_that_version_is_included() { - Set<Import> imports = calculateImports(referencedPackages, emptySet(), exportsWithVersion); - - assertThat(imports, contains(importMatching(PACKAGE_NAME, "1.3"))); + Set<Import> imports = calculateImports(referencedPackages, Set.of(), exportsWithVersion); + assertEquals(1, imports.stream().filter(imp -> imp.packageName().equals(PACKAGE_NAME) && imp.version().equals("1.3")).count()); } @Test public void require_that_all_versions_up_to_the_next_major_version_is_in_range() { - assertThat(new Import("foo", Optional.of("1.2")).importVersionRange().get(), is("[1.2,2)")); + assertEquals("[1.2,2)", new Import("foo", Optional.of("1.2")).importVersionRange().get()); } // TODO: Detecting guava packages should be based on bundle-symbolicName, not package name. @Test public void require_that_for_guava_all_future_major_versions_are_in_range() { Optional<String> rangeWithInfiniteUpperLimit = Optional.of("[18.1," + ImportPackages.INFINITE_VERSION + ")"); - assertThat(new Import("com.google.common", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); - assertThat(new Import("com.google.common.foo", Optional.of("18.1")).importVersionRange(), is(rangeWithInfiniteUpperLimit)); - assertThat(new Import("com.google.commonality", Optional.of("18.1")).importVersionRange(), is(Optional.of("[18.1,19)"))); + assertEquals(rangeWithInfiniteUpperLimit, new Import("com.google.common", Optional.of("18.1")).importVersionRange()); + assertEquals(rangeWithInfiniteUpperLimit, new Import("com.google.common.foo", Optional.of("18.1")).importVersionRange()); + assertEquals(Optional.of("[18.1,19)"), new Import("com.google.commonality", Optional.of("18.1")).importVersionRange()); } @Test public void require_that_none_version_gives_non_version_range() { - assertThat(new Import("foo", Optional.empty()).importVersionRange(), is(Optional.empty())); + assertTrue(new Import("foo", Optional.empty()).importVersionRange().isEmpty()); } @Rule @@ -95,18 +87,17 @@ public class ImportPackageTest { @Test public void require_that_osgi_import_supports_missing_version() { - assertThat(new Import("com.yahoo.exported", Optional.empty()).asOsgiImport(), is("com.yahoo.exported")); + assertEquals("com.yahoo.exported", new Import("com.yahoo.exported", Optional.empty()).asOsgiImport()); } @Test public void require_that_osgi_import_version_range_includes_all_versions_from_the_current_up_to_the_next_major_version() { - assertThat(new Import("com.yahoo.exported", Optional.of("1.2")).asOsgiImport(), is("com.yahoo.exported;version=\"[1.2,2)\"")); + assertEquals("com.yahoo.exported;version=\"[1.2,2)\"", new Import("com.yahoo.exported", Optional.of("1.2")).asOsgiImport()); } @Test public void require_that_osgi_import_version_range_ignores_qualifier() { - assertThat(new Import("com.yahoo.exported", Optional.of("1.2.3.qualifier")).asOsgiImport(), - is("com.yahoo.exported;version=\"[1.2.3,2)\"")); + assertEquals("com.yahoo.exported;version=\"[1.2.3,2)\"", new Import("com.yahoo.exported", Optional.of("1.2.3.qualifier")).asOsgiImport()); } private static Set<Import> calculateImports(Set<String> referencedPackages, @@ -114,18 +105,4 @@ public class ImportPackageTest { Map<String, Export> exportedPackages) { return new HashSet<>(ImportPackages.calculateImports(referencedPackages, implementedPackages, exportedPackages).values()); } - - private static TypeSafeMatcher<Import> importMatching(String packageName, String version) { - return new TypeSafeMatcher<Import>() { - @Override - protected boolean matchesSafely(Import anImport) { - return anImport.packageName().equals(packageName) && anImport.version().equals(version); - } - - @Override - public void describeTo(Description description) { - description.appendText("an Import of package ").appendValue(packageName).appendText(" with version ").appendValue(version); - } - }; - } } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index da50fadbd16..83434209615 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -43,7 +43,7 @@ <hk2.version>2.5.0-b32</hk2.version> <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> <httpclient5.version>5.1.2</httpclient5.version> - <jackson2.version>2.12.1</jackson2.version> + <jackson2.version>2.12.6</jackson2.version> <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javassist.version>3.20.0-GA</javassist.version> <javax.annotation-api.version>1.2</javax.annotation-api.version> @@ -219,7 +219,6 @@ <include>com.ibm.icu:icu4j:57.1:jar:test</include> <include>com.intellij:annotations:12.0:jar:test</include> <include>com.microsoft.onnxruntime:onnxruntime:[${onnxruntime.version}]:jar:test</include> - <include>com.optimaize.languagedetector:language-detector:0.6:jar:test</include> <include>com.thaiopensource:jing:20091111:jar:test</include> <include>com.yahoo.athenz:athenz-auth-core:[${athenz.version}]:jar:test</include> <include>com.yahoo.athenz:athenz-client-common:[${athenz.version}]:jar:test</include> @@ -249,7 +248,7 @@ <include>org.apache.httpcomponents.core5:httpcore5-h2:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents:httpclient:4.5.12:jar:test</include> <include>org.apache.httpcomponents:httpcore:4.4.13:jar:test</include> - <include>org.apache.opennlp:opennlp-tools:1.8.4:jar:test</include> + <include>org.apache.opennlp:opennlp-tools:1.9.3:jar:test</include> <include>org.apiguardian:apiguardian-api:1.1.0:jar:test</include> <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> <include>org.eclipse.jetty.alpn:alpn-api:[${jetty-alpn.version}]:jar:test</include> @@ -275,7 +274,7 @@ <include>org.kohsuke:libpam4j:1.11:jar:test</include> <include>org.opentest4j:opentest4j:1.2.0:jar:test</include> <include>software.amazon.ion:ion-java:1.0.2:jar:test</include> - <include>xerces:xercesImpl:2.12.0:jar:test</include> + <include>xerces:xercesImpl:2.12.1:jar:test</include> </includes> </bannedDependencies> </rules> diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 2a70799ab3d..ec68ed73864 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -17,10 +16,10 @@ import java.io.FileReader; import java.io.IOException; import java.nio.file.Files; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Ulf Lilleengen @@ -30,9 +29,6 @@ public class FilesApplicationPackageTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void testPreprocessing() throws IOException { File appDir = temporaryFolder.newFolder(); @@ -112,9 +108,12 @@ public class FilesApplicationPackageTest { IOUtils.copyDirectory(new File("src/test/resources/multienvapp"), appDir); Files.delete(new File(appDir, "services.xml").toPath()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("services.xml in application package is empty")); - app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); + try { + app.preprocess(new Zone(Environment.dev, RegionName.defaultName()), new BaseDeployLogger()); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("services.xml in application package is empty")); + } } } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index 7b2c2757b56..613cfe1cced 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -26,14 +26,10 @@ import static com.yahoo.test.FunctionTestConfig.RootStruct; import static com.yahoo.test.FunctionTestConfig.MyStructMap; import static com.yahoo.foo.MaptypesConfig.Innermap; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.startsWith; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -51,8 +47,8 @@ public class ConfigInstanceBuilderTest .gender(FEMALE); StructtypesConfig config = builder.build(); - assertThat(config.simple().name(), is("myname")); - assertThat(config.simple().gender(), is(FEMALE)); + assertEquals("myname", config.simple().name()); + assertEquals(FEMALE, config.simple().gender()); } @Test @@ -65,8 +61,8 @@ public class ConfigInstanceBuilderTest builder.intmap(newMap); MaptypesConfig config = new MaptypesConfig(builder); - assertThat(config.intmap("one"), is(1)); - assertThat(config.intmap("two"), is(2)); + assertEquals(1, config.intmap("one")); + assertEquals(2, config.intmap("two")); } @Test @@ -80,8 +76,8 @@ public class ConfigInstanceBuilderTest builder.innermap(newMap); MaptypesConfig config = new MaptypesConfig(builder); - assertThat(config.innermap("one").foo(), is(1)); - assertThat(config.innermap("two").foo(), is(2)); + assertEquals(1, config.innermap("one").foo()); + assertEquals(2, config.innermap("two").foo()); } @Test @@ -262,12 +258,12 @@ public class ConfigInstanceBuilderTest assertEquals("etc", config.fileVal().value()); assertEquals(1, config.boolarr().size()); assertEquals(1, config.boolarr().size()); // new api with accessor for a List of the original Java type - assertEquals(false, config.boolarr().get(0)); // new List api - assertEquals(false, config.boolarr(0)); // short-hand + assertFalse(config.boolarr().get(0)); // new List api + assertFalse(config.boolarr(0)); // short-hand assertEquals(0, config.intarr().size()); assertEquals(2, config.longarr().size()); assertEquals(Long.MAX_VALUE, config.longarr(0)); - assertThat(config.longarr().get(1), is(Long.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, config.longarr().get(1).longValue()); assertEquals(2, config.doublearr().size()); assertEquals(1, config.stringarr().size()); assertEquals(1, config.enumarr().size()); @@ -279,25 +275,25 @@ public class ConfigInstanceBuilderTest assertEquals("parent:", config.refarr(2)); assertEquals("bin", config.fileArr(0).value()); - assertThat(config.intMap("one"), is(1)); - assertThat(config.intMap("two"), is(2)); - assertThat(config.stringMap("one"), is("first")); - assertThat(config.filemap("f1").value(), is("/var")); - assertThat(config.filemap("f2").value(), is("/store")); + assertEquals(1, config.intMap("one")); + assertEquals(2, config.intMap("two")); + assertEquals("first", config.stringMap("one")); + assertEquals("/var", config.filemap("f1").value()); + assertEquals("/store", config.filemap("f2").value()); assertEquals("basicFoo", config.basicStruct().foo()); assertEquals(3, config.basicStruct().bar()); // new List api assertEquals(2, config.basicStruct().intArr().size()); - assertThat(config.basicStruct().intArr().get(0), is(310)); // new List api - assertThat(config.basicStruct().intArr().get(1), is(311)); // new List api + assertEquals(310, config.basicStruct().intArr().get(0).intValue()); // new List api + assertEquals(311, config.basicStruct().intArr().get(1).intValue()); // new List api assertEquals(310, config.basicStruct().intArr(0)); // short-hand assertEquals("inner0", config.rootStruct().inner0().name()); // new List api assertEquals(11, config.rootStruct().inner0().index()); assertEquals("inner1", config.rootStruct().inner1().name()); assertEquals(12, config.rootStruct().inner1().index()); assertEquals(2, config.rootStruct().innerArr().size()); - assertEquals(true, config.rootStruct().innerArr(0).boolVal()); + assertTrue(config.rootStruct().innerArr(0).boolVal()); assertEquals("deep", config.rootStruct().innerArr(0).stringVal()); - assertEquals(false, config.rootStruct().innerArr(1).boolVal()); + assertFalse(config.rootStruct().innerArr(1).boolVal()); assertEquals("blue a=\"escaped\"", config.rootStruct().innerArr(1).stringVal()); assertEquals(2, config.myarray().size()); // new List api @@ -311,12 +307,12 @@ public class ConfigInstanceBuilderTest assertEquals(-1, config.myarray(1).myStruct().a()); assertEquals(-2, config.myarray(1).myStruct().b()); - assertThat(config.myStructMap("one").myInt(), is(1)); - assertThat(config.myStructMap("one").myString(), is("bull")); - assertThat(config.myStructMap("one").myIntDef(), is(2)); - assertThat(config.myStructMap("one").myStringDef(), is("bear")); - assertThat(config.myStructMap("one").anotherMap("anotherOne").anInt(), is(3)); - assertThat(config.myStructMap("one").anotherMap("anotherOne").anIntDef(), is(4)); + assertEquals(1, config.myStructMap("one").myInt()); + assertEquals("bull", config.myStructMap("one").myString()); + assertEquals(2, config.myStructMap("one").myIntDef()); + assertEquals("bear", config.myStructMap("one").myStringDef()); + assertEquals(3, config.myStructMap("one").anotherMap("anotherOne").anInt()); + assertEquals(4, config.myStructMap("one").anotherMap("anotherOne").anIntDef()); } private boolean callContainsFieldsFlaggedWithRestart(Class<?> configClass) @@ -372,9 +368,8 @@ public class ConfigInstanceBuilderTest report = callGetChangesRequiringRestart(function1, function2); assertTrue(report.needsRestart()); assertEquals("function-test", report.getName()); - assertThat( - report.toString(), - startsWith( + assertTrue( + report.toString().startsWith( "# An int value\n" + "# Also test that multiline comments\n" + "# work.\n" + @@ -398,9 +393,8 @@ public class ConfigInstanceBuilderTest ) ); - assertThat( - report.toString(), - containsString( + assertTrue( + report.toString().contains( "function-test.myStructMap{one}.myInt has changed from 1 to 42\n" + "function-test.myStructMap{new} was added with value \n" ) @@ -410,9 +404,8 @@ public class ConfigInstanceBuilderTest FunctionTestConfig function3 = new FunctionTestConfig(funcBuilder); report = callGetChangesRequiringRestart(function2, function3); assertEquals(1, report.getReportLines().size()); - assertThat( - report.toString(), - containsString("function-test.myStructMap{one} with value \n") + assertTrue( + report.toString().contains("function-test.myStructMap{one} with value \n") ); } } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index 1ca33b8628a..536c18786da 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -14,10 +14,8 @@ import static com.yahoo.test.FunctionTestConfig.Enum_val; import static com.yahoo.test.FunctionTestConfig.Enumarr; import static com.yahoo.test.FunctionTestConfig.Myarray; import static com.yahoo.test.FunctionTestConfig.RootStruct; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; public class ConfigInstanceEqualsTest { FunctionTestConfig config1; @@ -33,90 +31,90 @@ public class ConfigInstanceEqualsTest { @Test public void require_same_hashCode_for_equal_instances() { - assertThat(config1.hashCode(), is(config2.hashCode())); + assertEquals(config1.hashCode(), config2.hashCode()); } @Test public void require_true_for_equal_instances() { - assertThat(config1, is(config2)); + assertEquals(config1, config2); } @Test public void require_false_for_null() { - assertThat(config1, not((FunctionTestConfig) null)); + assertNotEquals(null, config1); } @Test public void require_false_for_different_subclass() { - assertFalse(config1.equals(new AppConfig(new AppConfig.Builder()))); + assertNotEquals(config1, new AppConfig(new AppConfig.Builder())); } @Test public void require_false_for_different_scalars_at_root_node() { - assertThat(config1, not(new FunctionTestConfig(newBuilder().bool_val(true)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().int_val(0)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().long_val(0L)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().double_val(0.0)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().string_val("")))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().enum_val(Enum_val.FOO)))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().refval("")))); - assertThat(config1, not(new FunctionTestConfig(newBuilder().fileVal("")))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().bool_val(true))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().int_val(0))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().long_val(0L))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().double_val(0.0))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().string_val(""))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().enum_val(Enum_val.FOO))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().refval(""))); + assertNotEquals(config1, new FunctionTestConfig(newBuilder().fileVal(""))); } @Test public void require_false_for_different_leaf_array_at_root_node() { builder2.longarr.set(0, 0L); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_scalar_in_struct() { builder2.basicStruct(new BasicStruct.Builder(config1.basicStruct()).bar(0)); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_scalar_in_inner_array() { builder2.myarray.get(0).intval(0); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_false_for_different_leaf_array_in_inner_array() { builder2.myarray.get(0).stringval.set(0, ""); - assertThat(config1, not(new FunctionTestConfig(builder2))); + assertNotEquals(config1, new FunctionTestConfig(builder2)); } @Test public void require_equal_structs_for_equal_configs() { - assertThat(config1.basicStruct(), is(config2.basicStruct())); - assertThat(config1.rootStruct(), is(config2.rootStruct())); - assertThat(config1.rootStruct().inner0(), is(config2.rootStruct().inner0())); + assertEquals(config1.basicStruct(), config2.basicStruct()); + assertEquals(config1.rootStruct(), config2.rootStruct()); + assertEquals(config1.rootStruct().inner0(), config2.rootStruct().inner0()); } @Test public void require_equal_inner_arrays_for_equal_configs() { - assertThat(config1.myarray(), is(config2.myarray())); - assertThat(config1.myarray(0).anotherarray(), is(config2.myarray(0).anotherarray())); + assertEquals(config1.myarray(), config2.myarray()); + assertEquals(config1.myarray(0).anotherarray(), config2.myarray(0).anotherarray()); } @Test public void require_equal_inner_array_elements_for_equal_configs() { - assertThat(config1.myarray(0), is(config2.myarray(0))); - assertThat(config1.myarray(0).anotherarray(0), is(config2.myarray(0).anotherarray(0))); + assertEquals(config1.myarray(0), config2.myarray(0)); + assertEquals(config1.myarray(0).anotherarray(0), config2.myarray(0).anotherarray(0)); } @Test public void require_equal_leaf_arrays_for_equal_configs() { - assertThat(config1.intarr(), is(config2.intarr())); - assertThat(config1.boolarr(), is(config2.boolarr())); - assertThat(config1.longarr(), is(config2.longarr())); - assertThat(config1.doublearr(), is(config2.doublearr())); - assertThat(config1.stringarr(), is(config2.stringarr())); - assertThat(config1.enumarr(), is(config2.enumarr())); - assertThat(config1.refarr(), is(config2.refarr())); - assertThat(config1.fileArr(), is(config2.fileArr())); + assertEquals(config1.intarr(), config2.intarr()); + assertEquals(config1.boolarr(), config2.boolarr()); + assertEquals(config1.longarr(), config2.longarr()); + assertEquals(config1.doublearr(), config2.doublearr()); + assertEquals(config1.stringarr(), config2.stringarr()); + assertEquals(config1.enumarr(), config2.enumarr()); + assertEquals(config1.refarr(), config2.refarr()); + assertEquals(config1.fileArr(), config2.fileArr()); } private static FunctionTestConfig.Builder newBuilder() { diff --git a/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java b/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java index 42819957a1d..1192cc6673e 100644 --- a/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/DoubleNodeTest.java @@ -3,8 +3,10 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author Ulf Lilleengen diff --git a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java index b9ebaf1f4dd..fac765f40aa 100644 --- a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java @@ -3,8 +3,10 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -31,10 +33,10 @@ public class EnumNodeTest { public void testEnumNode() { MyNode n = new MyNode(); assertNull(n.getValue()); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); assertTrue(n.doSetValue("ONE")); - assertThat(n.getValue(), is("ONE")); - assertThat(n.toString(), is("ONE")); + assertEquals("ONE", n.getValue()); + assertEquals("ONE", n.toString()); assertFalse(n.doSetValue("THREE")); } } diff --git a/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java b/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java index 67edf5e9631..56dd7dd116d 100644 --- a/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/FileNodeTest.java @@ -3,9 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -16,11 +14,11 @@ public class FileNodeTest { @Test public void testSetValue() { FileNode n = new FileNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); assertTrue(n.doSetValue("foo.txt")); - assertThat(n.value().value(), is("foo.txt")); + assertEquals("foo.txt", n.value().value()); assertTrue(n.doSetValue("\"foo.txt\"")); - assertThat(n.value().value(), is("foo.txt")); - assertThat(n.toString(), is("\"foo.txt\"")); + assertEquals("foo.txt", n.value().value()); + assertEquals("\"foo.txt\"", n.toString()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java b/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java index c8404b8791e..899e34edce4 100644 --- a/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/IntegerNodeTest.java @@ -3,9 +3,8 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -18,6 +17,6 @@ public class IntegerNodeTest { IntegerNode n = new IntegerNode(); assertFalse(n.setValue("invalid")); assertTrue(n.setValue("10")); - assertThat(n.value(), is(10)); + assertEquals(10, n.value().intValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java b/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java index c3f9ff9ae8f..fc603ba65ac 100644 --- a/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/LongNodeTest.java @@ -3,9 +3,8 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -18,6 +17,6 @@ public class LongNodeTest { LongNode n = new LongNode(); assertFalse(n.setValue("invalid")); assertTrue(n.setValue("10")); - assertThat(n.value(), is(10l)); + assertEquals(10L, n.value().longValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java index cb9bd65c406..9637ab25278 100644 --- a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java +++ b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java @@ -5,9 +5,8 @@ import org.junit.Test; import java.util.Arrays; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -61,10 +60,10 @@ public class NodeVectorTest { StringNode val = new StringNode("foo"); TestNodeVector v = new TestNodeVector(val.getValue()); assertFalse(v.isEmpty()); - assertThat(v.indexOf(val), is(0)); - assertThat(v.indexOf(barNode()), is(-1)); - assertThat(v.lastIndexOf(val), is(0)); - assertThat(v.lastIndexOf(barNode()), is(-1)); + assertEquals(0, v.indexOf(val)); + assertEquals(-1, v.indexOf(barNode())); + assertEquals(0, v.lastIndexOf(val)); + assertEquals(-1, v.lastIndexOf(barNode())); } @Test @@ -82,11 +81,11 @@ public class NodeVectorTest { public void require_that_sublisting_works() { String val = "foo"; TestNodeVector v = new TestNodeVector(val, val, val); - assertThat(v.subList(0, 1).size(), is(1)); - assertThat(v.subList(0, 2).size(), is(2)); - assertThat(v.subList(0, 3).size(), is(3)); + assertEquals(1, v.subList(0, 1).size()); + assertEquals(2, v.subList(0, 2).size()); + assertEquals(3, v.subList(0, 3).size()); StringNode[] vals = v.toArray(new StringNode[0]); - assertThat(vals.length, is(3)); + assertEquals(3, vals.length); } private StringNode barNode() { return new StringNode("bar");} diff --git a/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java b/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java index 7bb465f203b..37313bbcdf3 100644 --- a/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/PathNodeTest.java @@ -5,8 +5,7 @@ import org.junit.Test; import java.io.File; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -17,10 +16,10 @@ public class PathNodeTest { @Test public void testSetValue() { PathNode n = new PathNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); n = new PathNode(new FileReference("foo.txt")); - assertThat(n.value(), is(new File("foo.txt").toPath())); + assertEquals(new File("foo.txt").toPath(), n.value()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java index 05b9361865c..5d606e12454 100644 --- a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java @@ -3,8 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author hmusum @@ -15,11 +14,11 @@ public class StringNodeTest { @Test public void testUnescapeQuotedString() { String a = "\"Hei\""; - assertThat(StringNode.unescapeQuotedString(a), is("Hei")); - assertThat(StringNode.unescapeQuotedString("foo\"bar\""), is("foo\"bar\"")); - assertThat(StringNode.unescapeQuotedString("foo\\\"bar\\\""), is("foo\"bar\"")); - assertThat(StringNode.unescapeQuotedString("a\\rb\\tc\\fd"), is("a\rb\tc\fd")); - assertThat(StringNode.unescapeQuotedString("\\x55"), is("U")); + assertEquals("Hei", StringNode.unescapeQuotedString(a)); + assertEquals("foo\"bar\"", StringNode.unescapeQuotedString("foo\"bar\"")); + assertEquals("foo\"bar\"", StringNode.unescapeQuotedString("foo\\\"bar\\\"")); + assertEquals("a\rb\tc\fd", StringNode.unescapeQuotedString("a\\rb\\tc\\fd")); + assertEquals("U", StringNode.unescapeQuotedString("\\x55")); } @Test(expected = IllegalArgumentException.class) @@ -30,17 +29,17 @@ public class StringNodeTest { @Test public void testToString() { StringNode n = new StringNode(); - assertThat(n.toString(), is("(null)")); + assertEquals("(null)", n.toString()); n.setValue("foo"); - assertThat(n.toString(), is("\"foo\"")); + assertEquals("\"foo\"", n.toString()); } @Test public void testSetValue() { StringNode n = new StringNode(); n.setValue("\"foo\""); - assertThat(n.getValue(), is("foo")); + assertEquals("foo", n.getValue()); n.setValue("foo"); - assertThat(n.getValue(), is("foo")); + assertEquals("foo", n.getValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java b/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java index 040e3304a66..abf04c34bb8 100644 --- a/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/UrlNodeTest.java @@ -3,8 +3,7 @@ package com.yahoo.config; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author lesters @@ -14,13 +13,13 @@ public class UrlNodeTest { @Test public void testSetValue() { UrlNode url = new UrlNode(); - assertThat(url.toString(), is("(null)")); + assertEquals("(null)", url.toString()); url = new UrlNode(new UrlReference("https://docs.vespa.ai/")); - assertThat(url.getUrlReference().value(), is("https://docs.vespa.ai/")); + assertEquals("https://docs.vespa.ai/", url.getUrlReference().value()); url = new UrlNode(new UrlReference("pom.xml")); - assertThat(url.getValue(), is("pom.xml")); + assertEquals("pom.xml", url.getValue()); } } diff --git a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java index 066b5383621..54e2271f7dc 100644 --- a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java +++ b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java @@ -6,8 +6,7 @@ import com.github.myproject.PackageConfig; import com.yahoo.my.namespace.NamespaceConfig; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -18,17 +17,17 @@ public class NamespaceAndPackageTest { @Test public void namespace_is_set_from_def_file() { - assertThat(NamespaceConfig.CONFIG_DEF_NAMESPACE, is(NAMESPACE)); + assertEquals(NAMESPACE, NamespaceConfig.CONFIG_DEF_NAMESPACE); } @Test public void package_is_used_as_namespace_when_namespace_is_not_set_explicitly() { - assertThat(PackageConfig.CONFIG_DEF_NAMESPACE, is(PACKAGE)); + assertEquals(PACKAGE, PackageConfig.CONFIG_DEF_NAMESPACE); } @Test public void package_does_not_override_namespace() { - assertThat(NamespaceAndPackageConfig.CONFIG_DEF_NAMESPACE, is(NAMESPACE)); + assertEquals(NAMESPACE, NamespaceAndPackageConfig.CONFIG_DEF_NAMESPACE); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 1f78ad20e40..6c70af8cbca 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } + @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index cf649162c08..e645fec5520 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -76,6 +76,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean unorderedMergeChaining = false; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; + private boolean failDeploymentWithInvalidJvmOptions = false; + private double tlsSizeFraction = 0.07; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -132,6 +134,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } + @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } + @Override public double tlsSizeFraction() { return tlsSizeFraction; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -347,6 +351,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties failDeploymentWithInvalidJvmOptions(boolean fail) { + failDeploymentWithInvalidJvmOptions = fail; + return this; + } + + public TestProperties tlsSizeFraction(double tlsSizeFraction) { + this.tlsSizeFraction = tlsSizeFraction; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index e8be43fdc96..c4d420f2d44 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -167,11 +167,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage; } - /** - * Returns the percentage of host physical memory this application has specified for nodes in this cluster, - * or empty if this is not specified by the application. - */ - public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } + @Override + public Optional<Integer> getMemoryPercentage() { + if (memoryPercentage != null) { + return Optional.of(memoryPercentage); + } else if (isHostedVespa()) { + return getHostClusterId().isPresent() ? + Optional.of(heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster) : + Optional.of(heapSizePercentageOfTotalNodeMemory); + } + return Optional.empty(); + } /* Create list of endpoints, these will be consumed later by the LBservicesProducer @@ -291,10 +297,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .heapsize(1536); if (getMemoryPercentage().isPresent()) { builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); - } else if (isHostedVespa()) { - builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? - heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster : - heapSizePercentageOfTotalNodeMemory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index a9ce03d1d55..c73a3b2a676 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -653,4 +653,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> public boolean getDeferChangesUntilRestart() { return deferChangesUntilRestart; } + /** + * Returns the percentage of host physical memory this application has specified for nodes in this cluster, + * or empty if this is not specified by the application. + */ + public Optional<Integer> getMemoryPercentage() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 6d184666bdb..c8b2197fc29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -93,6 +93,7 @@ import org.w3c.dom.Node; import java.net.URI; import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -1126,26 +1127,49 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } + /** + * Validates JVM GC options and logs a warning if anyone of them has invalid syntax or is an option + * that is unsupported for the running system (e.g. uses CMS options for hosted Vespa, which uses JDK 17) + */ private static class JvmGcOptions { + private static final Pattern validPattern = Pattern.compile("-XX:[+-]*[a-zA-z0-9=]+"); + private static final Pattern invalidCMSPattern = Pattern.compile("-XX:[+-]\\w*CMS[a-zA-z0-9=]+"); + private final DeployState deployState; private final String jvmGcOptions; private final DeployLogger logger; private final boolean isHosted; + private final boolean failDeploymentWithInvalidJvmOptions; public JvmGcOptions(DeployState deployState, String jvmGcOptions) { this.deployState = deployState; this.jvmGcOptions = jvmGcOptions; this.logger = deployState.getDeployLogger(); this.isHosted = deployState.isHosted(); + this.failDeploymentWithInvalidJvmOptions = deployState.featureFlags().failDeploymentWithInvalidJvmOptions(); } private String build() { String options = deployState.getProperties().jvmGCOptions(); if (jvmGcOptions != null) { - log(jvmGcOptions); options = jvmGcOptions; - // TODO: Verify options against lists of allowed and/or disallowed options + String[] optionList = options.split(" "); + List<String> invalidOptions = Arrays.stream(optionList) + .filter(option -> !option.isEmpty()) + .filter(option -> !Pattern.matches(validPattern.pattern(), option)) + .collect(Collectors.toList()); + + if (isHosted) { + // CMS GC options cannot be used in hosted, CMS is unsupported in JDK 17 + invalidOptions.addAll(Arrays.stream(optionList) + .filter(option -> !option.isEmpty()) + .filter(option -> Pattern.matches(invalidCMSPattern.pattern(), option) || + option.equals("-XX:+UseConcMarkSweepGC")) + .collect(Collectors.toList())); + } + + logOrFailInvalidOptions(invalidOptions); } if (options == null || options.isEmpty()) @@ -1154,9 +1178,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return options; } - private void log(String jvmGcOptions) { - if (isHosted) - logger.logApplicationPackage(Level.INFO, "JVM GC options from services.xml: " + jvmGcOptions); + private void logOrFailInvalidOptions(List<String> options) { + if (options.isEmpty()) return; + + Collections.sort(options); + String message = "Invalid JVM GC options from services.xml: " + String.join(",", options); + if (failDeploymentWithInvalidJvmOptions) + throw new IllegalArgumentException(message); + else + logger.logApplicationPackage(WARNING, message); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index f981ed6228f..54d09bacfa9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -74,21 +74,21 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final int defaultMaxCompactBuffers; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ - private final boolean combined; + private final double fractionOfMemoryReserved; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> { private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; - private final boolean combined; + private final double fractionOfMemoryReserved; private final ResourceLimits resourceLimits; public Builder(Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean combined, ResourceLimits resourceLimits) { + double fractionOfMemoryReserved, ResourceLimits resourceLimits) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.resourceLimits = resourceLimits; } @@ -106,7 +106,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> globallyDistributedDocuments, getFlushOnShutdown(flushOnShutdownElem), syncTransactionLog, - combined); + fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -208,7 +208,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> Set<NewDocumentType> globallyDistributedDocuments, boolean flushOnShutdown, Boolean syncTransactionLog, - boolean combined) + double fractionOfMemoryReserved) { super(parent, "search"); this.clusterName = clusterName; @@ -217,7 +217,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.flushOnShutdown = flushOnShutdown; this.syncTransactionLog = syncTransactionLog; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); this.feedTaskLimit = featureFlags.feedTaskLimit(); this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); @@ -290,7 +290,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> Optional<Tuning> tuning = Optional.ofNullable(this.tuning); if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, - clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), combined); + clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), + fractionOfMemoryReserved, deployState.featureFlags().tlsSizeFraction()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); @@ -298,7 +299,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, combined).build(deployState, parent, element.getXml()); + searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved) + .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } searchNode.setTls(tls); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 0c0b2b80d79..628cf6bb4c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -131,7 +131,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments, - isCombined(getClusterId(contentElement), containers), + fractionOfMemoryReserved(getClusterId(contentElement), containers), resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); @@ -240,12 +240,15 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } } - /** Returns whether this hosts one of the given container clusters */ - private boolean isCombined(String clusterId, Collection<ContainerModel> containers) { - return containers.stream() - .map(model -> model.getCluster().getHostClusterId()) - .filter(Optional::isPresent) - .anyMatch(id -> id.get().equals(clusterId)); + /** Returns of memory reserved on a host. Memory is reserved for the jvm if th ecluster is combined */ + private double fractionOfMemoryReserved(String clusterId, Collection<ContainerModel> containers) { + for (ContainerModel containerModel : containers) { + Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); + if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { + return containerModel.getCluster().getMemoryPercentage().get() * 0.01; + } + } + return 0.0; } private void setupExperimental(ContentCluster cluster, ModelElement experimental) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 65efcf85b1d..3af0b49de82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import static java.lang.Long.min; +import static java.lang.Long.max; /** * Tuning of proton config for a search node based on the resources on the node. @@ -20,17 +20,20 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L; private final NodeResources resources; private final int threadsPerSearch; - private final boolean combined; + private final double fractionOfMemoryReserved; + private final double tlsSizeFraction; // "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy). public static final double reservedMemoryGb = 0.5; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - boolean combined) { + double fractionOfMemoryReserved, + double tlsSizeFraction) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.tlsSizeFraction = tlsSizeFraction; } @Override @@ -95,8 +98,8 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB); - tlsSizeBytes = min(tlsSizeBytes, 100 * GB); + long tlsSizeBytes = (long) ((resources.diskGb() * tlsSizeFraction) * GB); + tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } @@ -122,12 +125,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { double usableMemoryGb = resources.memoryGb() - reservedMemoryGb; - if ( ! combined) { - return usableMemoryGb; - } - - double fractionTakenByContainer = ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster * 1e-2; - return usableMemoryGb * (1 - fractionTakenByContainer); + return usableMemoryGb * (1 - fractionOfMemoryReserved); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 4b3b464f9e5..31513a273b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -67,9 +67,8 @@ public class SearchNode extends AbstractService implements private AbstractService serviceLayerService; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - - /** Whether this search node is co-located with a container node on a hosted system */ - private final boolean combined; + private final double fractionOfMemoryReserved; + private final double tlsSizeFraction; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -80,11 +79,11 @@ public class SearchNode extends AbstractService implements private final boolean flushOnShutdown; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - private boolean combined; + private final double fractionOfMemoryReserved; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - boolean combined) { + double fractionOfMemoryReserved) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -92,13 +91,14 @@ public class SearchNode extends AbstractService implements this.flushOnShutdown = flushOnShutdown; this.tuning = tuning; this.resourceLimits = resourceLimits; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; } @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), combined); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved, + deployState.featureFlags().tlsSizeFraction()); } } @@ -106,16 +106,16 @@ public class SearchNode extends AbstractService implements public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { - return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, - flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); + double fractionOfMemoryReserved, double tlsSizeFraction) { + return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); + double fractionOfMemoryReserved, double tlsSizeFraction) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); @@ -123,11 +123,12 @@ public class SearchNode extends AbstractService implements private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - boolean combined) { + double fractionOfMemoryReserved, double tlsSizeFraction) { super(parent, name); setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; - this.combined = combined; + this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.tlsSizeFraction = tlsSizeFraction; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -281,7 +282,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - combined); + fractionOfMemoryReserved, tlsSizeFraction); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 171958f2bfa..912161019fe 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -278,6 +278,43 @@ public class ModelProvisioningTest { } } + @Test + public void testCombinedClusterWithJvmHeapSizeOverride() { + var containerElements = Set.of("jdisc", "container"); + for (var containerElement : containerElements) { + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <" + containerElement + " version='1.0' id='container1'>" + + " <search/>" + + " <nodes of='content1'>" + + " <jvm allocated-memory=\"30%\"/>" + + " </nodes>" + + " </" + containerElement + ">" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(5); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is lowered with combined clusters", + 30, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.30)), protonMemorySize(model.getContentClusters().get("content1"))); + assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); + } + } + /** For comparison with the above */ @Test public void testNonCombinedCluster() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 560ac28b6f7..bcd29267277 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -40,7 +40,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.OptionalInt; -import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; @@ -51,9 +50,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.applic import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasKey; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -100,8 +97,8 @@ public class ContainerClusterTest { assertEquals("cd", config.system()); } - private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster) { - return createContainerCluster(root, isCombinedCluster, null); + private ApplicationContainerCluster createContainerCluster(MockRoot root) { + return createContainerCluster(root, false, null); } private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster, Integer memoryPercentage) { ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", root.getDeployState()); @@ -162,7 +159,7 @@ public class ContainerClusterTest { private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) { MockRoot root = createRoot(isHosted); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); if (hasDocProc) { cluster.setDocproc(new ContainerDocproc(cluster, null)); } @@ -227,7 +224,7 @@ public class ContainerClusterTest { public void requireThatJvmOmitStackTraceInFastThrowOptionWorks() { // Empty option if option not set in property MockRoot root = createRoot(new DeployState.Builder().build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); ApplicationContainer container = cluster.getContainers().get(0); assertEquals("", container.getJvmOptions()); @@ -235,7 +232,7 @@ public class ContainerClusterTest { String jvmOption = "-XX:-foo"; DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build(); root = createRoot(deployState); - cluster = createContainerCluster(root, false); + cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); container = cluster.getContainers().get(0); assertEquals(jvmOption, container.getJvmOptions()); @@ -244,7 +241,7 @@ public class ContainerClusterTest { @Test public void requireThatWeCanHandleNull() { MockRoot root = createRoot(false); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); Container container = cluster.getContainers().get(0); container.setJvmOptions(""); @@ -256,7 +253,7 @@ public class ContainerClusterTest { @Test public void requireThatNonHostedUsesExpectedDefaultThreadpoolConfiguration() { MockRoot root = new MockRoot("foo"); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); @@ -268,13 +265,13 @@ public class ContainerClusterTest { @Test public void container_cluster_has_default_threadpool_provider() { MockRoot root = new MockRoot("foo"); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); ComponentId expectedComponentId = new ComponentId("default-threadpool"); var components = cluster.getComponentsMap(); - assertThat(components, hasKey(expectedComponentId)); + assertTrue(components.containsKey(expectedComponentId)); Component<?, ?> component = components.get(expectedComponentId); assertEquals(ThreadPoolProvider.class.getName(), component.getClassId().getName()); } @@ -287,7 +284,7 @@ public class ContainerClusterTest { .properties(new TestProperties().setHostedVespa(true)) .applicationPackage(new MockApplicationPackage.Builder().build()) .build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); @@ -304,7 +301,7 @@ public class ContainerClusterTest { .properties(new TestProperties().setHostedVespa(true)) .applicationPackage(new MockApplicationPackage.Builder().build()) .build()); - ApplicationContainerCluster cluster = createContainerCluster(root, false); + ApplicationContainerCluster cluster = createContainerCluster(root); addContainer(root, cluster, "c1", "host-c1"); root.freezeModelTopology(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index a674a06d45e..5fe2baaca37 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -9,17 +9,22 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; - import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.w3c.dom.Element; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.logging.Level; +import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,6 +35,8 @@ import static org.junit.Assert.assertTrue; */ public class JvmOptionsTest extends ContainerModelBuilderTestBase { + @Rule public ExpectedException expectedException = ExpectedException.none(); + @Test public void verify_jvm_tag_with_attributes() throws IOException, SAXException { String servicesXml = @@ -145,35 +152,76 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatJvmGcOptionsAreLogged() throws IOException, SAXException { - verifyLoggingOfJvmOptions(true, "gc-options", "-XX:+UseCMSInitiatingOccupancyOnly foo bar"); - verifyLoggingOfJvmOptions(true, "gc-options", "-XX:+UseConcMarkSweepGC"); - verifyLoggingOfJvmOptions(false, "gc-options", "-XX:+UseConcMarkSweepGC"); + public void requireThatInvalidJvmGcOptionsAreLogged() throws IOException, SAXException { + verifyLoggingOfJvmGcOptions(true, + "-XX:+ParallelGCThreads=8 foo bar", + "foo", "bar"); + verifyLoggingOfJvmGcOptions(true, + "-XX:+UseCMSInitiatingOccupancyOnly foo bar", + "-XX:+UseCMSInitiatingOccupancyOnly", "foo", "bar"); + verifyLoggingOfJvmGcOptions(true, + "-XX:+UseConcMarkSweepGC", + "-XX:+UseConcMarkSweepGC"); + verifyLoggingOfJvmGcOptions(true, + "$(touch /tmp/hello-from-gc-options)", + "$(touch", "/tmp/hello-from-gc-options)"); + + verifyLoggingOfJvmGcOptions(false, + "$(touch /tmp/hello-from-gc-options)", + "$(touch", "/tmp/hello-from-gc-options)"); + + // Valid options, should not log anything + verifyLoggingOfJvmGcOptions(true, "-XX:+ParallelGCThreads=8"); + verifyLoggingOfJvmGcOptions(true, "-XX:MaxTenuringThreshold"); // No + or - after colon + verifyLoggingOfJvmGcOptions(false, "-XX:+UseConcMarkSweepGC"); + } + + @Test + public void requireThatInvalidJvmGcOptionsFailDeployment() throws IOException, SAXException { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage(containsString("Invalid JVM GC options from services.xml: bar,foo")); + buildModelWithJvmOptions(new TestProperties().setHostedVespa(true).failDeploymentWithInvalidJvmOptions(true), + new TestLogger(), + "gc-options", + "-XX:+ParallelGCThreads=8 foo bar"); + } + + private void verifyLoggingOfJvmGcOptions(boolean isHosted, String override, String... invalidOptions) throws IOException, SAXException { + verifyLoggingOfJvmOptions(isHosted, "gc-options", override, invalidOptions); + } + + private void verifyLoggingOfJvmOptions(boolean isHosted, String optionName, String override, String... invalidOptions) throws IOException, SAXException { + TestLogger logger = new TestLogger(); + buildModelWithJvmOptions(isHosted, logger, optionName, override); + + List<String> strings = Arrays.asList(invalidOptions.clone()); + if (strings.isEmpty()) return; + + Collections.sort(strings); + Pair<Level, String> firstOption = logger.msgs.get(0); + assertEquals(Level.WARNING, firstOption.getFirst()); + assertEquals("Invalid JVM " + (optionName.equals("gc-options") ? "GC " : "") + + "options from services.xml: " + String.join(",", strings), firstOption.getSecond()); } - private void verifyLoggingOfJvmOptions(boolean isHosted, String optionName, String override) throws IOException, SAXException { + private void buildModelWithJvmOptions(boolean isHosted, TestLogger logger, String optionName, String override) throws IOException, SAXException { + buildModelWithJvmOptions(new TestProperties().setHostedVespa(isHosted), logger, optionName, override); + } + + private void buildModelWithJvmOptions(TestProperties properties, TestLogger logger, String optionName, String override) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + - " <nodes>" + - " <jvm " + optionName + "='" + override + "'/>" + - " <node hostalias='mockhost'/>" + - " </nodes>" + - "</container>"; + " <nodes>" + + " <jvm " + optionName + "='" + override + "'/>" + + " <node hostalias='mockhost'/>" + + " </nodes>" + + "</container>"; ApplicationPackage app = new MockApplicationPackage.Builder().withServices(servicesXml).build(); - TestLogger logger = new TestLogger(); new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(app) .deployLogger(logger) - .properties(new TestProperties().setHostedVespa(isHosted)) + .properties(properties) .build()); - if (isHosted) { - Pair<Level, String> firstOption = logger.msgs.get(0); - assertEquals(Level.INFO, firstOption.getFirst()); - assertEquals("JVM " + (optionName.equals("gc-options") ? "GC " : "") + - "options from services.xml: " + override, firstOption.getSecond()); - } else { - assertEquals(0, logger.msgs.size()); - } } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 357ae7e2656..d0515ae37e8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -47,6 +47,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; import java.util.OptionalInt; import static org.junit.Assert.assertEquals; @@ -1048,6 +1049,27 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } + private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional<Flavor> flavor) throws Exception { + TestProperties testProperties = new TestProperties(); + if (tlsSizeFraction.isPresent()) { + testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble()); + } + ContentCluster cc = createOneNodeCluster(testProperties, flavor); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + return protonConfig.flush().memory().maxtlssize(); + } + @Test + public void default_max_tls_size_controlled_by_properties() throws Exception { + var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); + assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty())); + assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty())); + assertEquals(7516192768L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor))); + assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor))); + assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor))); + } + void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 649a445e7a5..e2645e0b39e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.Test; import java.util.Arrays; @@ -31,8 +32,8 @@ public class NodeResourcesTuningTest { @Test public void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, false).hwinfo().memory().size()); - assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, true).hwinfo().memory().size(), 1000); + assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, 0).hwinfo().memory().size()); + assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).hwinfo().memory().size(), 1000); } @Test @@ -132,11 +133,13 @@ public class NodeResourcesTuningTest { @Test public void require_that_flush_strategy_tls_size_is_set_based_on_available_disk() { - assertFlushStrategyTlsSize(7 * GB, 100); - assertFlushStrategyTlsSize(35 * GB, 500); - assertFlushStrategyTlsSize(84 * GB, 1200); - assertFlushStrategyTlsSize(100 * GB, 1720); - assertFlushStrategyTlsSize(100 * GB, 24000); + assertFlushStrategyTlsSize(2 * GB, 10, 0.05); + assertFlushStrategyTlsSize(7 * GB, 100, 0.07); + assertFlushStrategyTlsSize(5 * GB, 100, 0.05); + assertFlushStrategyTlsSize(35 * GB, 500, 0.07); + assertFlushStrategyTlsSize(84 * GB, 1200, 0.07); + assertFlushStrategyTlsSize(100 * GB, 1720, 0.07); + assertFlushStrategyTlsSize(100 * GB, 24000, 0.07); } @Test @@ -153,14 +156,14 @@ public class NodeResourcesTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, false).summary().cache().maxbytes()); - assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, false).summary().cache().maxbytes()); + assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, 0).summary().cache().maxbytes()); + assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, 0).summary().cache().maxbytes()); } @Test public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { - assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); - assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); } @Test @@ -169,16 +172,16 @@ public class NodeResourcesTuningTest { } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) { - assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).summary().log().maxfilesize()); + assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).summary().log().maxfilesize()); } private static void assertFlushStrategyMemory(long expMemoryBytes, int wantedMemoryGb) { - assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().maxmemory()); - assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().each().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory()); } - private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { - assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize()); + private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb, double tlsSizeFraction) { + assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb, tlsSizeFraction).flush().memory().maxtlssize()); } private static void assertSummaryReadIo(ProtonConfig.Summary.Read.Io.Enum expValue, boolean fastDisk) { @@ -194,55 +197,67 @@ public class NodeResourcesTuningTest { } private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, false).writefilter().memorylimit(), delta); + assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta); } private static ProtonConfig configFromDiskSetting(boolean fastDisk) { - return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), false); + return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk)); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), false); + return configFromDiskSetting(diskGb, 0.07); + } + private static ProtonConfig configFromDiskSetting(int diskGb, double tlsSizeFraction) { + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0, tlsSizeFraction); } - private static ProtonConfig configFromMemorySetting(double memoryGb, boolean combined) { - return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined); + private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) { + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved, 0.07); } private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { return getConfig(new FlavorsConfig.Flavor.Builder() - .minMainMemoryAvailableGb(memoryGb), builder, false); + .minMainMemoryAvailableGb(memoryGb), builder); } private static ProtonConfig configFromNumCoresSetting(double numCores) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), false); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); } private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) { return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), - new ProtonConfig.Builder(), numThreadsPerSearch, false); + new ProtonConfig.Builder(), numThreadsPerSearch); } private static ProtonConfig configFromEnvironmentType(boolean docker) { String environment = (docker ? "DOCKER_CONTAINER" : "undefined"); - return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), false); + return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment)); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, boolean combined) { - return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { + return getConfig(flavorBuilder, new ProtonConfig.Builder()); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, boolean combined) { - flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), 1, combined); - tuning.getConfig(protonBuilder); - return new ProtonConfig(protonBuilder); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved, tlsSizeFraction); + } + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { + return getConfig(flavorBuilder, protonBuilder,1); + } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { + return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved, tlsSizeFraction); + } + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int numThreadsPerSearch) { + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, 0.07); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, boolean combined) { + int numThreadsPerSearch, double fractionOfMemoryReserved, double tlsSizeFraction) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, combined); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved, tlsSizeFraction); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index cca2d817df4..9077fcea4d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -52,12 +51,14 @@ public class SchemaNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, - NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean combined) { - return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); + NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { + return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, + Optional.empty(), Optional.empty(), isHosted, 0.0, + root.getDeployState().featureFlags().tlsSizeFraction()); } private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false); + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true); } @Test @@ -70,7 +71,7 @@ public class SchemaNodeTest { @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); prepare(root, node, true); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -78,7 +79,7 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.deployLogger()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -87,7 +88,7 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.deployLogger()); assertTrue(node.getPreShutdownCommand().isPresent()); diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index 12b31bd6370..5f0c93ceec5 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -64,11 +64,6 @@ Provisioning APIs. <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java index 27b9a7213cb..c82230f7edf 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ApplicationIdTest.java @@ -7,8 +7,6 @@ import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.test.TotalOrderTester; import org.junit.Test; import com.google.common.testing.EqualsTester; -import static org.junit.Assert.assertThat; -import static org.hamcrest.Matchers.*; /** * @author Ulf Lilleengen @@ -53,9 +51,9 @@ public class ApplicationIdTest { ApplicationId id1 = applicationId("foo"); ApplicationId id2 = applicationId("bar"); ApplicationId id3 = idFrom("tenant", "baz", "bim"); - assertThat(id1.serializedForm(), is("default:foo:default")); - assertThat(id2.serializedForm(), is("default:bar:default")); - assertThat(id3.serializedForm(), is("tenant:baz:bim")); + assertEquals("default:foo:default", id1.serializedForm()); + assertEquals("default:bar:default", id2.serializedForm()); + assertEquals("tenant:baz:bim", id3.serializedForm()); } @Test @@ -63,20 +61,20 @@ public class ApplicationIdTest { ApplicationId id1 = applicationId("foo"); ApplicationId id2 = idFrom("bar", "baz", "default"); ApplicationId id3 = idFrom("tenant", "baz", "bim"); - assertThat(id1.toShortString(), is("default.foo")); - assertThat(id1.toFullString(), is("default.foo.default")); - assertThat(id2.toShortString(), is("bar.baz")); - assertThat(id2.toFullString(), is("bar.baz.default")); - assertThat(id3.toShortString(), is("tenant.baz.bim")); - assertThat(id3.toFullString(), is("tenant.baz.bim")); + assertEquals("default.foo", id1.toShortString()); + assertEquals("default.foo.default", id1.toFullString()); + assertEquals("bar.baz", id2.toShortString()); + assertEquals("bar.baz.default", id2.toFullString()); + assertEquals("tenant.baz.bim", id3.toShortString()); + assertEquals("tenant.baz.bim", id3.toFullString()); } @Test public void require_that_idstring_can_be_parsed() { ApplicationId id = ApplicationId.fromSerializedForm("ten:foo:bim"); - assertThat(id.tenant().value(), is("ten")); - assertThat(id.application().value(), is("foo")); - assertThat(id.instance().value(), is("bim")); + assertEquals("ten", id.tenant().value()); + assertEquals("foo", id.application().value()); + assertEquals("bim", id.instance().value()); } @Test(expected = IllegalArgumentException.class) @@ -87,8 +85,8 @@ public class ApplicationIdTest { @Test public void require_that_defaults_are_given() { ApplicationId id1 = applicationId("foo"); - assertThat(id1.tenant().value(), is("default")); - assertThat(id1.instance().value(), is("default")); + assertEquals("default", id1.tenant().value()); + assertEquals("default", id1.instance().value()); } @Test diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java index 370bcd6cdf1..d4edb7a14eb 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/IdentifierTestBase.java @@ -4,10 +4,9 @@ package com.yahoo.config.provision; import com.google.common.testing.EqualsTester; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -29,8 +28,8 @@ public abstract class IdentifierTestBase<ID_TYPE> { assertTrue(isDefault(def)); assertTrue(isDefault(def2)); assertFalse(isDefault(notdef)); - assertThat(def, is(def2)); - assertThat(def2, is(not(notdef))); + assertEquals(def, def2); + assertNotEquals(def2, notdef); } @Test diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java index 8475c30c1f6..2bfcd183c85 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/TenantTest.java @@ -4,8 +4,7 @@ package com.yahoo.config.provision; import com.yahoo.test.TotalOrderTester; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * @author Ulf Lilleengen @@ -29,7 +28,7 @@ public class TenantTest extends IdentifierTestBase<TenantName> { @Test public void testComparator() { - assertThat(TenantName.defaultName().compareTo(TenantName.defaultName()), is(0)); + assertEquals(0, TenantName.defaultName().compareTo(TenantName.defaultName())); new TotalOrderTester<TenantName>() .theseObjects(TenantName.from("a"), TenantName.from("a")) diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 5f3445d1d8f..604c85555db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -67,8 +67,8 @@ public class JRTConfigRequester implements RequestWaiter { this.scheduler = scheduler; this.connectionPool = connectionPool; this.timingValues = timingValues; - // Adjust so that we wait 1 second with logging warning in case there are some errors just when starting up - timeForLastLogWarning = Instant.now().minus(delayBetweenWarnings.plus(Duration.ofSeconds(5))); + // Adjust so that we wait 5 seconds with logging warning in case there are some errors just when starting up + timeForLastLogWarning = Instant.now().minus(delayBetweenWarnings).plus(Duration.ofSeconds(5)); } /** diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java index 05e2e7c6ff6..9cf363376cc 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefLineParsingTest.java @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + /** * Tests parsing of a single line of a .def file * @@ -225,9 +227,9 @@ public class DefLineParsingTest { DefLine i = new DefLine("i int range=[0, 100]"); DefLine l = new DefLine("l long range=[-1e10, 0]"); DefLine d = new DefLine("d double range=[0, 1.0]"); - assertThat(i.getRange(), is("[0, 100]")); - assertThat(l.getRange(), is("[-1e10, 0]")); - assertThat(d.getRange(), is("[0, 1.0]")); + assertEquals("[0, 100]", i.getRange()); + assertEquals("[-1e10, 0]", l.getRange()); + assertEquals("[0, 1.0]", d.getRange()); } @Test diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java index 97ee17cb6d5..7921a3aecbe 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java @@ -8,9 +8,8 @@ import java.io.IOException; import static com.yahoo.config.codegen.DefParserTest.assertLineFails; import static com.yahoo.config.codegen.DefParserTest.createDefTemplate; import static com.yahoo.config.codegen.DefParserTest.createParser; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; /** * @author gjoranv @@ -22,7 +21,7 @@ public class DefParserNamespaceTest { public void namespace_is_set_on_root_node() { DefParser parser = createParser("version=1\nnamespace=myproject.config\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is("myproject.config")); + assertEquals("myproject.config", root.getNamespace()); } @Test @@ -30,7 +29,7 @@ public class DefParserNamespaceTest { String PACKAGE = "com.github.myproject"; DefParser parser = createParser("package=" + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is(PACKAGE)); + assertEquals(PACKAGE, root.getNamespace()); } @Test(expected = CodegenRuntimeException.class) @@ -48,7 +47,7 @@ public class DefParserNamespaceTest { public void spaces_are_allowed_around_equals_sign() { DefParser parser = createParser("version=1\nnamespace = myproject.config\n"); CNode root = parser.getTree(); - assertThat(root.getNamespace(), is("myproject.config")); + assertEquals("myproject.config", root.getNamespace()); } @Test @@ -69,7 +68,7 @@ public class DefParserNamespaceTest { parser = createParser("version=1\nnamespace=myproject.config\n"); CNode namespaceRoot = parser.getTree(); - assertThat(root.defMd5, not(namespaceRoot.defMd5)); + assertNotEquals(root.defMd5, namespaceRoot.defMd5); } @@ -82,12 +81,12 @@ public class DefParserNamespaceTest { } @Test - public void number_is_not_allowed_as_namespace_start_char() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_namespace_start_char() { assertLineFails("namespace=2.a.b"); } @Test - public void number_is_not_allowed_as_leading_char_in_namespace_token() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_leading_char_in_namespace_token() { assertLineFails("namespace=a.b.2c"); } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java index d995ab63886..cd7ca1a89b9 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserPackageTest.java @@ -5,14 +5,12 @@ import org.junit.Test; import java.io.IOException; -import static com.yahoo.config.codegen.DefParser.DEFAULT_PACKAGE_PREFIX; import static com.yahoo.config.codegen.DefParserTest.assertLineFails; import static com.yahoo.config.codegen.DefParserTest.createDefTemplate; import static com.yahoo.config.codegen.DefParserTest.createParser; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; /** * Tests setting explicit java package in the def file. @@ -26,7 +24,7 @@ public class DefParserPackageTest { public void package_is_set_on_root_node() { DefParser parser = createParser("package=" + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); + assertEquals(PACKAGE, root.getPackage()); } @Test @@ -35,8 +33,8 @@ public class DefParserPackageTest { DefParser parser = createParser("package=" + PACKAGE + "\nnamespace=" + namespace +"\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); - assertThat(root.getNamespace(), is(namespace)); + assertEquals(PACKAGE, root.getPackage()); + assertEquals(namespace, root.getNamespace()); } // Required by JavaClassBuilder ctor. @@ -45,7 +43,7 @@ public class DefParserPackageTest { String namespace = "test.namespace"; DefParser parser = createParser("namespace=" + namespace + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), nullValue()); + assertNull(root.getPackage()); } @Test(expected = CodegenRuntimeException.class) @@ -57,7 +55,7 @@ public class DefParserPackageTest { public void spaces_are_allowed_around_equals_sign() { DefParser parser = createParser("package = " + PACKAGE + "\n"); CNode root = parser.getTree(); - assertThat(root.getPackage(), is(PACKAGE)); + assertEquals(PACKAGE, root.getPackage()); } @Test @@ -78,7 +76,7 @@ public class DefParserPackageTest { parser = createParser("package=" + PACKAGE + "\na string\n"); CNode rootWithPackage = parser.getTree(); - assertThat(root.defMd5, not(rootWithPackage.defMd5)); + assertNotEquals(root.defMd5, rootWithPackage.defMd5); } @@ -91,12 +89,12 @@ public class DefParserPackageTest { } @Test - public void number_is_not_allowed_as_package_start_char() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_package_start_char() { assertLineFails("package=2.a.b"); } @Test - public void number_is_not_allowed_as_leading_char_in_package_token() throws IOException, DefParser.DefParserException { + public void number_is_not_allowed_as_leading_char_in_package_token() { assertLineFails("package=a.b.2c"); } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java index 22ccf4105c1..e4d9f6cb3bd 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java @@ -1,15 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Test; import org.junit.Ignore; -import static org.hamcrest.CoreMatchers.is; - -import java.io.*; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; /** * Unit tests for DefParser. @@ -28,7 +31,7 @@ public class DefParserTest { CNode root = new DefParser("test", new FileReader(defFile)).getTree(); assertNotNull(root); CNode[] children = root.getChildren(); - assertThat(children.length, is(34)); + assertEquals(34, children.length); int numGrandChildren = 0; int numGreatGrandChildren = 0; @@ -39,45 +42,45 @@ public class DefParserTest { numGreatGrandChildren += grandChild.getChildren().length; } } - assertThat(numGrandChildren, is(14)); - assertThat(numGreatGrandChildren, is(6)); + assertEquals(14, numGrandChildren); + assertEquals(6, numGreatGrandChildren); // Verify that each array creates a sub-tree, and that defaults for leafs are handled correctly. CNode myArray = root.getChild("myArray"); - assertThat(myArray.getChildren().length, is(5)); + assertEquals(5, myArray.getChildren().length); // int within array LeafCNode myArrayInt = (LeafCNode) myArray.getChild("intVal"); - assertThat(myArrayInt.getDefaultValue().getValue(), is("14")); + assertEquals("14", myArrayInt.getDefaultValue().getValue()); // enum within array LeafCNode myArrayEnum = (LeafCNode) myArray.getChild("enumVal"); - assertThat(myArrayEnum.getDefaultValue().getValue(), is("TYPE")); + assertEquals("TYPE", myArrayEnum.getDefaultValue().getValue()); // Verify array within array and a default value for a leaf in the inner array. CNode anotherArray = myArray.getChild("anotherArray"); - assertThat(anotherArray.getChildren().length, is(1)); + assertEquals(1, anotherArray.getChildren().length); LeafCNode foo = (LeafCNode) anotherArray.getChild("foo"); - assertThat(foo.getDefaultValue().getValue(), is("-4")); + assertEquals("-4", foo.getDefaultValue().getValue()); } @Test public void testFileWithNamespaceInFilename() throws IOException { File defFile = new File(TEST_DIR + "baz.bar.foo.def"); CNode root = new DefParser("test", new FileReader(defFile)).getTree(); - assertThat(root.defMd5, is("31a0f9bda0e5ff929762a29569575a7e")); + assertEquals("31a0f9bda0e5ff929762a29569575a7e", root.defMd5); } @Test public void testMd5Sum() throws IOException { File defFile = new File(DEF_NAME); CNode root = new DefParser("test", new FileReader(defFile)).getTree(); - assertThat(root.defMd5, is("f901bdc5c96e7005130399c63f247823")); + assertEquals("f901bdc5c96e7005130399c63f247823", root.defMd5); } @Test public void testMd5Sum2() { String def = "version=1\na string\n"; CNode root = new DefParser("testMd5Sum2", new StringReader(def)).getTree(); - assertThat(root.defMd5, is("a5e5fdbb2b27e56ba7d5e60e335c598b")); + assertEquals("a5e5fdbb2b27e56ba7d5e60e335c598b", root.defMd5); } @Test @@ -104,7 +107,7 @@ public class DefParserTest { private void testExpectedVersion(String versionLine, String expectedVersion) { InnerCNode root = createParser(versionLine).getTree(); - assertThat(root.defVersion, is(expectedVersion)); + assertEquals(expectedVersion, root.defVersion); } @Test @@ -146,7 +149,7 @@ public class DefParserTest { @Test(expected = CodegenRuntimeException.class) @Ignore("Not implemented yet") - public void testInvalidEnum() throws DefParser.DefParserException { + public void testInvalidEnum() { DefParser parser = createParser("version=1\nanEnum enum {A, B, A}\n"); //parser.validateDef(def); } @@ -171,7 +174,7 @@ public class DefParserTest { CNode root = parser.getTree(); LeafCNode node = (LeafCNode) root.getChild("enum1"); assertNotNull(node); - assertThat(node.getDefaultValue().getStringRepresentation(), is("A")); + assertEquals("A", node.getDefaultValue().getStringRepresentation()); } @Test(expected = DefParser.DefParserException.class) diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java index cce2d66b5f0..d69617ec5da 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java @@ -12,9 +12,7 @@ import java.util.List; import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static com.yahoo.config.codegen.JavaClassBuilder.createUniqueSymbol; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -82,8 +80,8 @@ public class JavaClassBuilderTest { "n int\n"; InnerCNode root = new DefParser("test", new StringReader(testDefinition)).getTree(); - assertThat(createUniqueSymbol(root, "foo"), is("f")); - assertThat(createUniqueSymbol(root, "name"), is("na")); + assertEquals("f", createUniqueSymbol(root, "foo")); + assertEquals("na", createUniqueSymbol(root, "name")); assertTrue(createUniqueSymbol(root, "m").startsWith(ReservedWords.INTERNAL_PREFIX + "m")); // The basis string is not a legal return value, even if unique, to avoid @@ -93,12 +91,12 @@ public class JavaClassBuilderTest { @Test public void testCreateClassName() { - assertThat(createClassName("simple"), is("SimpleConfig")); - assertThat(createClassName("a"), is("AConfig")); - assertThat(createClassName("a-b-c"), is("ABCConfig")); - assertThat(createClassName("a-1-2b"), is("A12bConfig")); - assertThat(createClassName("my-app"), is("MyAppConfig")); - assertThat(createClassName("MyApp"), is("MyAppConfig")); + assertEquals("SimpleConfig", createClassName("simple")); + assertEquals("AConfig", createClassName("a")); + assertEquals("ABCConfig", createClassName("a-b-c")); + assertEquals("A12bConfig", createClassName("a-1-2b")); + assertEquals("MyAppConfig", createClassName("my-app")); + assertEquals("MyAppConfig", createClassName("MyApp")); } @Test(expected = CodegenRuntimeException.class) diff --git a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java index 3c328721b6c..411ac0fe58a 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java @@ -1,14 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.junit.Assert.*; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; import java.util.List; import org.junit.Test; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** @@ -38,13 +42,13 @@ public class NormalizedDefinitionTest { } assertNotNull(out); - assertThat(out.size(), is(6)); - assertThat(out.get(0), is ("version=1\n")); - assertThat(out.get(1), is ("aString string\n")); - assertThat(out.get(2), is ("anInt int\n")); - assertThat(out.get(3), is ("aStringCommentCharacterAfter string default=\"ab\"\n")); - assertThat(out.get(4), is ("aStringWithCommentCharacter string default=\"a#b\"\n")); - assertThat(out.get(5), is ("aStringWithEscapedQuote string default=\"a\"b\"\n")); + assertEquals(6, out.size()); + assertEquals("version=1\n", out.get(0)); + assertEquals("aString string\n", out.get(1)); + assertEquals("anInt int\n", out.get(2)); + assertEquals("aStringCommentCharacterAfter string default=\"ab\"\n", out.get(3)); + assertEquals("aStringWithCommentCharacter string default=\"a#b\"\n", out.get(4)); + assertEquals("aStringWithEscapedQuote string default=\"a\"b\"\n", out.get(5)); reader.close(); } @@ -68,7 +72,7 @@ public class NormalizedDefinitionTest { } assertNotNull(out); - assertThat(out.size(), is(72)); + assertEquals(72, out.size()); assertNotNull(fileReader); fileReader.close(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 978241339d2..063603fe8a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,6 +203,8 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8GeoPositions; private final boolean useV8DocManagerCfg; private final int maxCompactBuffers; + private final boolean failDeploymentWithInvalidJvmOptions; + private final double tlsSizeFraction; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -244,6 +246,8 @@ public class ModelContextImpl implements ModelContext { this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); + this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); + this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -286,6 +290,9 @@ public class ModelContextImpl implements ModelContext { @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public boolean useV8DocManagerCfg() { return useV8DocManagerCfg; } + @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } + @Override public int maxCompactBuffers() { return maxCompactBuffers; } + @Override public double tlsSizeFraction() { return tlsSizeFraction; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java index 563c9a8bdff..a38f39559f5 100644 --- a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java +++ b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java @@ -18,10 +18,12 @@ import com.yahoo.language.opennlp.OpenNlpLinguistics; public class DefaultLinguisticsProvider implements Provider<Linguistics> { // Use lazy initialization to avoid expensive (memory-wise) instantiation - private final Supplier<Linguistics> linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); + private final Supplier<Linguistics> linguisticsSupplier; @Inject - public DefaultLinguisticsProvider() { } + public DefaultLinguisticsProvider() { + linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); + } @Override public Linguistics get() { return linguisticsSupplier.get(); } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index b34dcafacdf..ff6786080ef 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -424,7 +424,7 @@ <!-- and then verify by doing: ' ls -l vespa/vespa_jersey2/target/dependency' --> <hk2.version>2.5.0-b32</hk2.version> <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> - <jackson2.version>2.12.1</jackson2.version> + <jackson2.version>2.12.6</jackson2.version> <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javassist.version>3.20.0-GA</javassist.version> <javax.annotation-api.version>1.2</javax.annotation-api.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index a76f295a7b1..034081f4620 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -194,10 +194,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </exclusion> - <exclusion> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </exclusion> diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java index 40c66e115e4..70e956480ed 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/FilterBindingsProviderTest.java @@ -11,7 +11,12 @@ import com.yahoo.jdisc.http.filter.ResponseFilter; import com.yahoo.jdisc.http.server.jetty.FilterBindings; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -39,9 +44,9 @@ public class FilterBindingsProviderTest { final FilterBindings filterBindings = provider.get(); - assertThat(filterBindings).isNotNull(); - assertThat(filterBindings.requestFilterIds()).isEmpty(); - assertThat(filterBindings.responseFilterIds()).isEmpty(); + assertNotNull(filterBindings); + assertTrue(filterBindings.requestFilterIds().isEmpty()); + assertTrue(filterBindings.responseFilterIds().isEmpty()); } @Test @@ -92,11 +97,11 @@ public class FilterBindingsProviderTest { final FilterBindings filterBindings = provider.get(); // Verify. - assertThat(filterBindings).isNotNull(); - assertThat(filterBindings.requestFilters()) - .containsExactlyInAnyOrder(requestFilter1Instance, requestFilter2Instance); - assertThat(filterBindings.responseFilters()) - .containsExactlyInAnyOrder(responseFilter1Instance, responseFilter3Instance); + assertNotNull(filterBindings); + assertEquals(filterBindings.requestFilters().stream().collect(Collectors.toSet()), + Set.of(requestFilter1Instance, requestFilter2Instance)); + assertEquals(filterBindings.responseFilters().stream().collect(Collectors.toSet()), + Set.of(responseFilter1Instance, responseFilter3Instance)); } private interface DualRoleFilter extends RequestFilter, ResponseFilter {} @@ -127,7 +132,7 @@ public class FilterBindingsProviderTest { new ComponentRegistry<>()); fail("Dual-role filter should not be accepted"); } catch (RuntimeException e) { - assertThat(e.getMessage()).contains("Invalid config"); + assertTrue(e.getMessage().contains("Invalid config")); } } @@ -152,7 +157,7 @@ public class FilterBindingsProviderTest { new ComponentRegistry<>()); fail("Config with unknown filter reference should not be accepted"); } catch (RuntimeException e) { - assertThat(e.getMessage()).contains("Invalid config"); + assertTrue(e.getMessage().contains("Invalid config")); } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java index 4336c491071..549a1e3b887 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java @@ -31,8 +31,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -56,11 +55,11 @@ public class ClientThreadingTestCase { } final ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS); for (final Future<Boolean> res : executor.invokeAll(lst, 60, TimeUnit.SECONDS)) { - assertThat(res.get(), is(true)); + assertTrue(res.get()); } - assertThat(client.close(), is(true)); - assertThat(server.close(), is(true)); + assertTrue(client.close()); + assertTrue(server.close()); } private static final class RequestTask implements Callable<Boolean> { diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index 13059f7c91f..397a3eb5242 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java @@ -18,7 +18,6 @@ import com.yahoo.messagebus.shared.ServerSession; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; -import org.hamcrest.Matcher; import org.junit.Ignore; import org.junit.Test; @@ -31,12 +30,12 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import static com.yahoo.messagebus.ErrorCode.APP_FATAL_ERROR; -import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_CLOSED; import static com.yahoo.messagebus.ErrorCode.SESSION_BUSY; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -51,28 +50,28 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Test public void testContainerNotReadyException() throws Throwable { new TestRunner().setRequestTimeout(100, TimeUnit.MILLISECONDS) - .expectError(is(SESSION_BUSY)) + .expectError(SESSION_BUSY) .executeAndClose(); } @Override @Test public void testBindingSetNotFoundException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @Override @Test public void testNoBindingSetSelectedException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @Override @Test public void testBindingNotFoundException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -107,7 +106,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestException() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -142,7 +141,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestExceptionAfterResponseWriteWithSyncHandleResponse() throws Throwable { + public void testRequestExceptionAfterResponseWriteWithSyncHandleResponse() { } @Override @@ -154,7 +153,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testRequestExceptionBeforeResponseWriteWithAsyncHandleResponse() throws Throwable { - new TestRunner().expectError(is(APP_FATAL_ERROR)) + new TestRunner().expectError(APP_FATAL_ERROR) .executeAndClose(); } @@ -168,181 +167,181 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestExceptionAfterResponseWriteWithAsyncHandleResponse() throws Throwable { + public void testRequestExceptionAfterResponseWriteWithAsyncHandleResponse() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithNondeterministicSyncFailure() throws Throwable { + public void testRequestContentWriteWithNondeterministicSyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteWithSyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteWithSyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithNondeterministicAsyncFailure() throws Throwable { + public void testRequestContentWriteWithNondeterministicAsyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteWithAsyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteWithAsyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteWithAsyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteWithAsyncFailureAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicException() throws Throwable { + public void testRequestContentWriteNondeterministicException() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicExceptionWithSyncCompletion() throws Throwable { + public void testRequestContentWriteNondeterministicExceptionWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWriteWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithSyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithSyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteNondeterministicExceptionWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteNondeterministicExceptionWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionBeforeResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionBeforeResponseWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseWriteWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithAsyncCompletion() throws Throwable { + public void testRequestContentWriteExceptionAfterResponseCloseNoContentWithAsyncCompletion() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithNondeterministicSyncFailure() throws Throwable { + public void testRequestContentWriteExceptionWithNondeterministicSyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithSyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionWithSyncFailureAfterResponseCloseNoContent() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithNondeterministicAsyncFailure() throws Throwable { + public void testRequestContentWriteExceptionWithNondeterministicAsyncFailure() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureBeforeResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureBeforeResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseWrite() { } @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseCloseNoContent() throws Throwable { + public void testRequestContentWriteExceptionWithAsyncFailureAfterResponseCloseNoContent() { } @Override @@ -376,7 +375,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseWithSyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentCloseWithSyncFailureAfterResponseWrite() { } @Override @@ -403,7 +402,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseWithAsyncFailureAfterResponseWrite() throws Throwable { + public void testRequestContentCloseWithAsyncFailureAfterResponseWrite() { } @Override @@ -430,7 +429,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWrite() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWrite() { } @Override @@ -457,7 +456,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithSyncCompletion() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithSyncCompletion() { } @Override @@ -484,7 +483,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncCompletion() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncCompletion() { } @Override @@ -511,7 +510,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithSyncFailure() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithSyncFailure() { } @Override @@ -538,7 +537,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncFailure() throws Throwable { + public void testRequestContentCloseExceptionAfterResponseWriteWithAsyncFailure() { } @Override @@ -551,7 +550,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Override @Test @Ignore // N/A: The messagebus protocol does not have content. - public void testResponseWriteCompletionException() throws Throwable { + public void testResponseWriteCompletionException() { } @Override @@ -573,7 +572,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { final LocalWire wire = new LocalWire(); final SharedMessageBus mbus; final ServerSession session; - Matcher<Integer> expectedError = null; + Integer expectedError = null; boolean successExpected = false; long timeoutMillis = TimeUnit.SECONDS.toMillis(60); @@ -592,14 +591,14 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { return this; } - TestRunner expectError(Matcher<Integer> matcher) { - assertThat(successExpected, is(false)); + TestRunner expectError(Integer matcher) { + assertFalse(successExpected); expectedError = matcher; return this; } TestRunner expectSuccess() { - assertThat(expectedError, is(nullValue())); + assertNull(expectedError); successExpected = true; return this; } @@ -621,24 +620,24 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { } @Override - public MyClient newClient(MbusServer server) throws Throwable { + public MyClient newClient(MbusServer server) { return new MyClient(wire, server.connectionSpec()); } @Override public Reply executeRequest(MyClient client, boolean withRequestContent) throws Throwable { // This protocol doesn't have the concept of "request content", so if we are asked to send any, it's a bug. - assertThat(withRequestContent, is(false)); + assertFalse(withRequestContent); final SimpleMessage msg = new SimpleMessage("foo"); msg.getTrace().setLevel(9); msg.setRoute(client.route); msg.setTimeRemaining(timeoutMillis); - assertThat("client.session.send(msg).isAccepted()", - client.session.send(msg).isAccepted(), is(true)); + assertTrue("client.session.send(msg).isAccepted()", + client.session.send(msg).isAccepted()); final Reply reply = client.replies.poll(60, TimeUnit.SECONDS); - assertThat("reply != null", reply, notNullValue()); + assertNotNull("reply != null", reply); return reply; } @@ -648,15 +647,15 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { } @Override - public void validateResponse(Reply reply) throws Throwable { + public void validateResponse(Reply reply) { final String trace = String.valueOf(reply.getTrace()); if (expectedError != null) { - assertThat(reply.hasErrors(), is(true)); + assertTrue(reply.hasErrors()); final int error = reply.getError(0).getCode(); - assertThat(trace, error, expectedError); + assertEquals(trace, Integer.valueOf(error), expectedError); } if (successExpected) { - assertThat(trace, reply.hasErrors(), is(false)); + assertFalse(trace, reply.hasErrors()); } } diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java index 0884762195b..0888a02c74c 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java @@ -28,9 +28,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult @@ -50,16 +49,16 @@ public class ServerThreadingTestCase { final Message msg = new SimpleMessage("foo"); msg.setRoute(Route.parse(server.delegate.connectionSpec())); msg.pushHandler(client); - assertThat(client.session.send(msg).isAccepted(), is(true)); + assertTrue(client.session.send(msg).isAccepted()); } for (int i = 0; i < NUM_REQUESTS; ++i) { final Reply reply = client.replies.poll(600, TimeUnit.SECONDS); - assertThat(reply, instanceOf(EmptyReply.class)); - assertThat(reply.hasErrors(), is(false)); + assertTrue(reply instanceof EmptyReply); + assertFalse(reply.hasErrors()); } - assertThat(client.close(), is(true)); - assertThat(server.close(), is(true)); + assertTrue(client.close()); + assertTrue(server.close()); } private static class Client implements ReplyHandler { @@ -106,14 +105,10 @@ public class ServerThreadingTestCase { @Override public void handleMessage(final Message msg) { - executor.execute(new Runnable() { - - @Override - public void run() { - final Reply reply = new EmptyReply(); - reply.swapState(msg); - reply.popHandler().handleReply(reply); - } + executor.execute(() -> { + final Reply reply = new EmptyReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); }); } diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 90c9c056131..e98b2a08662 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -217,11 +217,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java b/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java index 93668cf788d..317f5fc1329 100644 --- a/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java +++ b/container-search-and-docproc/src/test/java/com/yahoo/container/handler/observability/ApplicationStatusHandlerTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.processing.Processor; @@ -17,11 +16,9 @@ import org.junit.Test; import org.mockito.Mockito; import java.util.HashMap; -import java.util.concurrent.Executors; import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers; -import static org.junit.Assert.assertThat; -import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -30,7 +27,7 @@ import static org.hamcrest.CoreMatchers.containsString; public class ApplicationStatusHandlerTest { @Test - public void application_configs_are_rendered() throws Exception { + public void application_configs_are_rendered() { ApplicationMetadataConfig metaConfig = new ApplicationMetadataConfig( new ApplicationMetadataConfig.Builder() .checksum("abc") @@ -44,28 +41,28 @@ public class ApplicationStatusHandlerTest { .version("v1")); String json = ApplicationStatusHandler.renderApplicationConfigs(metaConfig, userConfig).toString(); - assertThat(json, containsString("version")); - assertThat(json, containsString("meta")); - assertThat(json, containsString("abc")); - assertThat(json, containsString("app")); - assertThat(json, containsString("/a/b/c")); - assertThat(json, containsString("3000")); - assertThat(json, containsString("donald")); - - assertThat(json, containsString("v1")); + assertTrue(json.contains("version")); + assertTrue(json.contains("meta")); + assertTrue(json.contains("abc")); + assertTrue(json.contains("app")); + assertTrue(json.contains("/a/b/c")); + assertTrue(json.contains("3000")); + assertTrue(json.contains("donald")); + + assertTrue(json.contains("v1")); } @Test - public void object_components_are_rendered() throws Exception { + public void object_components_are_rendered() { HashMap<ComponentId, Object> id2object = new HashMap<>(); id2object.put(new ComponentId("myComponent"), new Object()); String json = ApplicationStatusHandler.renderObjectComponents(id2object).toString(); - assertThat(json, containsString("myComponent")); + assertTrue(json.contains("myComponent")); } @Test - public void request_handlers_are_rendered() throws Exception { + public void request_handlers_are_rendered() { final String id = "myHandler"; final String serverBinding1 = "http://*/serverBinding"; final String serverBinding2 = "http://*/anotherServerBinding"; @@ -81,14 +78,14 @@ public class ApplicationStatusHandlerTest { .clientBindings(clientBinding)) ); String json = ApplicationStatusHandler.renderRequestHandlers(bindingsConfig, handlersById).toString(); - assertThat(json, containsString("\"" + id + "\"")); - assertThat(json, containsString(serverBinding1)); - assertThat(json, containsString(serverBinding2)); - assertThat(json, containsString(clientBinding)); + assertTrue(json.contains("\"" + id + "\"")); + assertTrue(json.contains(serverBinding1)); + assertTrue(json.contains(serverBinding2)); + assertTrue(json.contains(clientBinding)); } @Test - public void client_providers_are_rendered() throws Exception { + public void client_providers_are_rendered() { final String id = "myClient"; final String clientBinding = "http://*/clientBinding"; final String clientBinding2 = "http://*/anotherClientBinding"; @@ -105,21 +102,21 @@ public class ApplicationStatusHandlerTest { ); String json = ApplicationStatusHandler.renderRequestHandlers(bindingsConfig, clientsById).toString(); System.out.println(json); - assertThat(json, containsString("\"" + id + "\"")); - assertThat(json, containsString(clientBinding)); - assertThat(json, containsString(clientBinding2)); - assertThat(json, containsString(serverBinding)); + assertTrue(json.contains("\"" + id + "\"")); + assertTrue(json.contains(clientBinding)); + assertTrue(json.contains(clientBinding2)); + assertTrue(json.contains(serverBinding)); } @Test - public void chains_are_rendered() throws Exception { + public void chains_are_rendered() { ChainRegistry<Processor> chains = new ChainRegistry<>(); - Chain<Processor> chain = new Chain<Processor>("myChain", new VoidProcessor(new ComponentId("voidProcessor"))); + Chain<Processor> chain = new Chain<>("myChain", new VoidProcessor(new ComponentId("voidProcessor"))); chains.register(new ComponentId("myChain"), chain); String json = ApplicationStatusHandler.StatusResponse.renderChains(chains).toString(); - assertThat(json, containsString("myChain")); - assertThat(json, containsString("voidProcessor")); + assertTrue(json.contains("myChain")); + assertTrue(json.contains("voidProcessor")); } private static class VoidProcessor extends Processor { diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index 38efba3b511..8f4ed9d5ea3 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -174,7 +174,7 @@ keyword_as_ident : SELECT | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | OUTPUT | COUNT | SOURCES | MATCHES | LIKE ; -program : (statement SEMI)* EOF +program : (statement SEMI?)* EOF ; moduleId diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 6079c03ec5c..48dfa98768a 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -155,7 +155,7 @@ public class JSONSearchHandlerTestCase { assertTrue("Do I have a new instance of the search handler?", searchHandler != newSearchHandler); try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { ObjectNode json = jsonMapper.createObjectNode(); - json.put("yql", "select * from foo where bar > 1453501295"); + json.put("yql", "selectz * from foo where bar > 1453501295"); RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); responseHandler.readAll(); assertThat(responseHandler.getStatus(), is(400)); diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index 09d1dc71249..fdd3bf6e1c9 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -68,7 +68,7 @@ public class MinimalQueryInserterTestCase { assertEquals("foo", query.getPresentation().getSummaryFields().toArray(new String[1])[0]); builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + - "| all(group(a) each(output(count())));"); + "| all(group(a) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -76,7 +76,7 @@ public class MinimalQueryInserterTestCase { builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + "| all(group(a) each(output(count()))) " + - "| all(group(b) each(output(count())));"); + "| all(group(b) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -89,7 +89,7 @@ public class MinimalQueryInserterTestCase { URIBuilder builder = new URIBuilder(); builder.setPath("search/"); - builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + builder.setParameter("yql", "select foo from bar where baz contains 'cox'"); Query query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -97,7 +97,7 @@ public class MinimalQueryInserterTestCase { builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + - "all(group(a) each(output(count())));"); + "all(group(a) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -107,7 +107,7 @@ public class MinimalQueryInserterTestCase { "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + "all(group(a) each(output(count()))) " + "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + - "all(group(b) each(output(count())));"); + "all(group(b) each(output(count())))"); query = new Query(builder.toString()); execution.search(query); assertEquals("baz:cox", query.getModel().getQueryTree().toString()); @@ -119,14 +119,14 @@ public class MinimalQueryInserterTestCase { @Ignore // TODO: YQL work in progress (jon) public void testTmp() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); //execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); } @Test public void testSearch() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(Language.ENGLISH, query.getModel().getParsingLanguage()); @@ -135,7 +135,7 @@ public class MinimalQueryInserterTestCase { @Test public void testExplicitLanguageIsHonoredWithVerbatimQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?language=ja" + "&yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22%3B"); + Query query = new Query("search/?language=ja" + "&yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("title:"+ japaneseWord, query.getModel().getQueryTree().toString()); @@ -144,7 +144,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithVerbatimQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22%3B"); + Query query = new Query("search/?yql=select%20ignoredField%20from%20ignoredsource%20where%20title%20contains%20%22" + encode(japaneseWord) + "%22"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("title:"+ japaneseWord, query.getModel().getQueryTree().toString()); @@ -153,7 +153,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithUserInput() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?userString=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userInput(@userString)%3B"); + Query query = new Query("search/?userString=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userInput(@userString)"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("AND title:madonna default:" + japaneseWord, query.getModel().getQueryTree().toString()); @@ -162,7 +162,7 @@ public class MinimalQueryInserterTestCase { @Test public void testUserLanguageIsDetectedWithUserQuery() { String japaneseWord = "\u30ab\u30bf\u30ab\u30ca"; - Query query = new Query("search/?query=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=" + encode(japaneseWord) + "&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals(Language.JAPANESE, query.getModel().getParsingLanguage()); assertEquals("AND title:madonna " + japaneseWord, query.getModel().getQueryTree().toString()); @@ -170,14 +170,14 @@ public class MinimalQueryInserterTestCase { @Test public void testUserQueryFailsWithoutArgument() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); } @Test public void testSearchFromAllSourcesWithUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(0, query.getModel().getSources().size()); @@ -185,7 +185,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromAllSourcesWithoutUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(0, query.getModel().getSources().size()); @@ -193,7 +193,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromSomeSourcesWithoutUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(2, query.getModel().getSources().size()); @@ -203,7 +203,7 @@ public class MinimalQueryInserterTestCase { @Test public void testSearchFromSomeSourcesWithUserSource() { - Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(3, query.getModel().getSources().size()); @@ -214,7 +214,7 @@ public class MinimalQueryInserterTestCase { @Test public final void testSearchFromSomeSourcesWithOverlappingUserSource() { - final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()"); execution.search(query); assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); assertEquals(3, query.getModel().getSources().size()); @@ -225,7 +225,7 @@ public class MinimalQueryInserterTestCase { @Test public void testLimitAndOffset() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207"); execution.search(query); assertEquals(7, query.getOffset()); assertEquals(24, query.getHits()); @@ -235,7 +235,7 @@ public class MinimalQueryInserterTestCase { @Test public void testMaxOffset() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000"); Result r = execution.search(query); assertEquals(1, r.hits().getErrorHit().errors().size()); ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); @@ -245,7 +245,7 @@ public class MinimalQueryInserterTestCase { @Test public void testMaxLimit() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207"); Result r = execution.search(query); assertEquals(1, r.hits().getErrorHit().errors().size()); ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); @@ -255,7 +255,7 @@ public class MinimalQueryInserterTestCase { @Test public void testTimeout() { - Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051%3B"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051"); execution.search(query); assertEquals(51L, query.getTimeout()); assertEquals("select * from sources * where title contains \"madonna\" timeout 51;", query.yqlRepresentation()); @@ -281,7 +281,7 @@ public class MinimalQueryInserterTestCase { @Test public void testOrdering() { { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals(2, query.getRanking().getSorting().fieldOrders() @@ -297,7 +297,7 @@ public class MinimalQueryInserterTestCase { assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", query.yqlRepresentation()); } { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals("other", query.getRanking().getSorting().fieldOrders() @@ -307,7 +307,7 @@ public class MinimalQueryInserterTestCase { assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3;", query.yqlRepresentation()); } { - String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203%3B"; + String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); { @@ -334,7 +334,7 @@ public class MinimalQueryInserterTestCase { @Test public void testStringRepresentation() { - String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 3eb6f5e7e38..981a79aa9fc 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -48,24 +48,21 @@ public class UserInputTestCase { public void testSimpleUserInput() { { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where userInput(\"nalle\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nalle\";", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); builder.setParameter("nalle", "bamse"); - builder.setParameter("yql", - "select * from sources * where userInput(@nalle);"); + builder.setParameter("yql", "select * from sources * where userInput(@nalle)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"bamse\";", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); builder.setParameter("nalle", "bamse"); - builder.setParameter("yql", - "select * from sources * where userInput(nalle);"); + builder.setParameter("yql", "select * from sources * where userInput(nalle)"); Query query = new Query(builder.toString()); Result r = execution.search(query); assertNotNull(r.hits().getError()); @@ -75,8 +72,7 @@ public class UserInputTestCase { @Test public void testRawUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"raw\"}]userInput(\"nal le\");"); + builder.setParameter("yql", "select * from sources * where [{\"grammar\": \"raw\"}]userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); } @@ -85,7 +81,7 @@ public class UserInputTestCase { public void testSegmentedUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"nal le\");"); + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains ([{\"origin\": {\"original\": \"nal le\", \"offset\": 0, \"length\": 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); } @@ -94,7 +90,7 @@ public class UserInputTestCase { public void testSegmentedNoiseUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"^^^^^^^^\");"); + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"^^^^^^^^\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); } @@ -103,7 +99,7 @@ public class UserInputTestCase { public void testCustomDefaultIndexUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"defaultIndex\": \"glompf\"}]userInput(\"nalle\");"); + "select * from sources * where [{\"defaultIndex\": \"glompf\"}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where glompf contains \"nalle\";", query.yqlRepresentation()); } @@ -112,7 +108,7 @@ public class UserInputTestCase { public void testAnnotatedUserInputStemming() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"stem\": false}]userInput(\"nalle\");"); + "select * from sources * where [{\"stem\": false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"stem\": false}]\"nalle\");", @@ -124,7 +120,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("myinput", "-5"); builder.setParameter("yql", - "select * from ecitem where rank(([{\"defaultIndex\":\"myfield\"}](userInput(@myinput))));"); + "select * from ecitem where rank(([{\"defaultIndex\":\"myfield\"}](userInput(@myinput))))"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from ecitem where rank(myfield = (-5));", query.yqlRepresentation()); assertEquals("RANK myfield:-5", query.getModel().getQueryTree().getRoot().toString()); @@ -134,7 +130,7 @@ public class UserInputTestCase { public void testAnnotatedUserInputUnrankedTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"ranked\": false}]userInput(\"nalle\");"); + "select * from sources * where [{\"ranked\": false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"ranked\": false}]\"nalle\");", @@ -145,7 +141,7 @@ public class UserInputTestCase { public void testAnnotatedUserInputFiltersTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"filter\": true}]userInput(\"nalle\");"); + "select * from sources * where [{\"filter\": true}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"filter\": true}]\"nalle\");", @@ -157,7 +153,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter( "yql", - "select * from sources * where [{\"normalizeCase\": false}]userInput(\"nalle\");"); + "select * from sources * where [{\"normalizeCase\": false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"normalizeCase\": false}]\"nalle\");", @@ -168,7 +164,7 @@ public class UserInputTestCase { public void testAnnotatedUserInputAccentRemoval() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"accentDrop\": false}]userInput(\"nalle\");"); + "select * from sources * where [{\"accentDrop\": false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"accentDrop\": false}]\"nalle\");", @@ -179,7 +175,7 @@ public class UserInputTestCase { public void testAnnotatedUserInputPositionData() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{\"usePositionData\": false}]userInput(\"nalle\");"); + "select * from sources * where [{\"usePositionData\": false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( "select * from sources * where default contains ([{\"usePositionData\": false}]\"nalle\");", @@ -192,7 +188,7 @@ public class UserInputTestCase { builder.setParameter("nalle", "bamse"); builder.setParameter("meta", "syntactic"); builder.setParameter("yql", - "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle);"); + "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle)"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"));", query.yqlRepresentation()); } @@ -201,7 +197,7 @@ public class UserInputTestCase { public void testReferenceInComparison() { URIBuilder builder = searchUri(); builder.setParameter("varref", "1980"); - builder.setParameter("yql", "select * from sources * where year > @varref;"); + builder.setParameter("yql", "select * from sources * where year > @varref"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation()); } @@ -212,7 +208,7 @@ public class UserInputTestCase { builder.setParameter("continuation", "BCBCBCBEBG"); builder.setParameter("yql", "select * from sources * where myfield contains 'token'" + - "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())));"); + "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())))"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where myfield contains \"token\" | [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); } @@ -222,7 +218,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("term", "A"); builder.setParameter("yql", - "select foo from bar where fieldName contains equiv(@term,'B');"); + "select foo from bar where fieldName contains equiv(@term,'B')"); Query query = searchAndAssertNoErrors(builder); assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\");", query.yqlRepresentation()); } @@ -249,7 +245,7 @@ public class UserInputTestCase { @Test public void testEmptyUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where userInput(\"\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"\")"); assertQueryFails(builder); } @@ -257,7 +253,7 @@ public class UserInputTestCase { public void testEmptyUserInputFromQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo)"); assertQueryFails(builder); } @@ -265,7 +261,7 @@ public class UserInputTestCase { public void testEmptyQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo);"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo)"); assertQueryFails(builder); } @@ -274,17 +270,17 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); builder.setParameter("yql", - "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo);"); + "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo)"); assertQueryFails(builder); } @Test public void testCompositeWithoutArguments() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase();"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase()"); searchAndAssertNoErrors(builder); builder = searchUri(); - builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase());"); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase())"); assertQueryFails(builder); } @@ -292,7 +288,7 @@ public class UserInputTestCase { public void testAnnoyingPlacementOfNonEmpty() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"));"); + "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"))"); assertQueryFails(builder); } @@ -305,7 +301,7 @@ public class UserInputTestCase { public void testAllowEmptyUserInput() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo)"); searchAndAssertNoErrors(builder); } @@ -313,7 +309,7 @@ public class UserInputTestCase { public void testAllowEmptyNullFromQueryParsing() { URIBuilder builder = searchUri(); builder.setParameter("foo", ",,,,,,,,"); - builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo)"); searchAndAssertNoErrors(builder); } @@ -321,7 +317,7 @@ public class UserInputTestCase { public void testDisallowEmptyNullFromQueryParsing() { URIBuilder builder = searchUri(); builder.setParameter("foo", ",,,,,,,,"); - builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo)"); assertQueryFails(builder); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 2a21214c702..0513179d10d 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -71,7 +71,7 @@ public class YqlParserTestCase { @Test(timeout = 120_000) public void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { - assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10;", + assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'œuvre') limit 10", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 token recognition error at: 'œ'")); } @@ -84,13 +84,13 @@ public class YqlParserTestCase { @Test public void testLanguageDetection() { // SimpleDetector used here can detect japanese and will set that as language at the root of the user input - QueryTree tree = parse("select * from sources * where userInput(\"\u30ab\u30bf\u30ab\u30ca\");"); + QueryTree tree = parse("select * from sources * where userInput(\"\u30ab\u30bf\u30ab\u30ca\")"); assertEquals(Language.JAPANESE, tree.getRoot().getLanguage()); } @Test public void testGroupingStep() { - assertParse("select foo from bar where baz contains 'cox';", + assertParse("select foo from bar where baz contains 'cox'", "baz:cox"); assertEquals("[]", toString(parser.getGroupingSteps())); @@ -113,14 +113,14 @@ public class YqlParserTestCase { @Test public void testGroupingContinuation() { assertParse("select foo from bar where baz contains 'cox' " + - "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())));", + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())))", "baz:cox"); assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", toString(parser.getGroupingSteps())); assertParse("select foo from bar where baz contains 'cox' " + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count()))) " + - "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())));", + "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())))", "baz:cox"); assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", @@ -194,8 +194,8 @@ public class YqlParserTestCase { @Test public void testLessThan() { - assertParse("select foo from bar where price < 500;", "price:<500"); - assertParse("select foo from bar where 500 < price;", "price:>500"); + assertParse("select foo from bar where price < 500", "price:<500"); + assertParse("select foo from bar where 500 < price", "price:>500"); } @Test @@ -206,8 +206,8 @@ public class YqlParserTestCase { @Test public void testLessThanOrEqual() { - assertParse("select foo from bar where price <= 500;", "price:[;500]"); - assertParse("select foo from bar where 500 <= price;", "price:[500;]"); + assertParse("select foo from bar where price <= 500", "price:[;500]"); + assertParse("select foo from bar where 500 <= price", "price:[500;]"); } @Test @@ -690,80 +690,72 @@ public class YqlParserTestCase { @Test public void testLongNumberInSimpleExpression() { - assertParse("select foo from bar where price = 8589934592L;", - "price:8589934592"); + assertParse("select foo from bar where price = 8589934592L", "price:8589934592"); } @Test public void testNegativeLongNumberInSimpleExpression() { - assertParse("select foo from bar where price = -8589934592L;", - "price:-8589934592"); + assertParse("select foo from bar where price = -8589934592L", "price:-8589934592"); } @Test public void testSources() { - assertSources("select foo from sourceA where price <= 500;", - Arrays.asList("sourceA")); + assertSources("select foo from sourceA where price <= 500", List.of("sourceA")); + } + + @Test + public void testQueryWithSemicolon() { + assertParse("select foo from bar where price = 1;", "price:1"); } @Test public void testSourcesWithDash() { - assertSources("select foo from source-a where price <= 500;", - Arrays.asList("source-a")); + assertSources("select foo from source-a where price <= 500", List.of("source-a")); } @Test public void testWildCardSources() { - assertSources("select foo from sources * where price <= 500;", - Collections.<String>emptyList()); + assertSources("select foo from sources * where price <= 500", List.of()); } @Test public void testMultiSources() { - assertSources("select foo from sources sourceA, sourceB where price <= 500;", - Arrays.asList("sourceA", "sourceB")); + assertSources("select foo from sources sourceA, sourceB where price <= 500", List.of("sourceA", "sourceB")); } @Test public void testFields() { - assertSummaryFields("select fieldA from bar where price <= 500;", - Arrays.asList("fieldA")); - assertSummaryFields("select fieldA, fieldB from bar where price <= 500;", - Arrays.asList("fieldA", "fieldB")); - assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500;", - Arrays.asList("fieldA", "fieldB", "fieldC")); - assertSummaryFields("select * from bar where price <= 500;", - Collections.<String>emptyList()); + assertSummaryFields("select fieldA from bar where price <= 500", List.of("fieldA")); + assertSummaryFields("select fieldA, fieldB from bar where price <= 500", List.of("fieldA", "fieldB")); + assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500", List.of("fieldA", "fieldB", "fieldC")); + assertSummaryFields("select * from bar where price <= 500", List.of()); } @Test public void testFieldsRoot() { - assertParse("select * from bar where price <= 500;", - "price:[;500]"); + assertParse("select * from bar where price <= 500", "price:[;500]"); } @Test public void testOffset() { - assertParse("select foo from bar where title contains \"madonna\" offset 37;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" offset 37", "title:madonna"); assertEquals(Integer.valueOf(37), parser.getOffset()); } @Test public void testLimit() { - assertParse("select foo from bar where title contains \"madonna\" limit 29;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" limit 29", "title:madonna"); assertEquals(Integer.valueOf(29), parser.getHits()); } @Test public void testOffsetAndLimit() { - assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29;", + assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29", "title:madonna"); assertEquals(Integer.valueOf(29), parser.getOffset()); assertEquals(Integer.valueOf(2), parser.getHits()); - assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37;", + assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37", "title:madonna"); assertEquals(Integer.valueOf(37), parser.getOffset()); assertEquals(Integer.valueOf(4), parser.getHits()); @@ -771,19 +763,17 @@ public class YqlParserTestCase { @Test public void testTimeout() { - assertParse("select * from bar where title contains \"madonna\" timeout 7;", - "title:madonna"); + assertParse("select * from bar where title contains \"madonna\" timeout 7", "title:madonna"); assertEquals(Integer.valueOf(7), parser.getTimeout()); - assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3;", - "title:madonna"); + assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3", "title:madonna"); assertEquals(Integer.valueOf(3), parser.getTimeout()); } @Test public void testOrdering() { assertParse("select foo from bar where title contains \"madonna\" order by something asc, " + - "shoesize desc limit 600 timeout 3;", + "shoesize desc limit 600 timeout 3", "title:madonna"); assertEquals(2, parser.getSorting().fieldOrders().size()); assertEquals("something", parser.getSorting().fieldOrders().get(0).getFieldName()); @@ -792,7 +782,7 @@ public class YqlParserTestCase { assertEquals(Order.DESCENDING, parser.getSorting().fieldOrders().get(1).getSortOrder()); assertParse("select foo from bar where title contains \"madonna\" order by other limit 600 " + - "timeout 3;", + "timeout 3", "title:madonna"); assertEquals("other", parser.getSorting().fieldOrders().get(0).getFieldName()); assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); @@ -803,7 +793,7 @@ public class YqlParserTestCase { assertParse( "select foo from bar where title contains \"madonna\"" + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc" - + " limit 600" + " timeout 3;", "title:madonna"); + + " limit 600" + " timeout 3", "title:madonna"); FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); assertEquals("other", fieldOrder.getFieldName()); assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); @@ -820,7 +810,7 @@ public class YqlParserTestCase { "select foo from bar where title contains \"madonna\"" + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc," + " [{\"function\": \"lowercase\"}]something asc" - + " limit 600" + " timeout 3;", "title:madonna"); + + " limit 600" + " timeout 3", "title:madonna"); { FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); assertEquals("other", fieldOrder.getFieldName()); @@ -842,29 +832,29 @@ public class YqlParserTestCase { @Test public void testSegmenting() { - assertParse("select * from bar where title contains 'foo.bar';", - "title:'foo bar'"); - - assertParse("select * from bar where title contains 'foo&123';", - "title:'foo 123'"); + assertParse("select * from bar where title contains 'foo.bar'", "title:'foo bar'"); + assertParse("select * from bar where title contains 'foo&123'", "title:'foo 123'"); } @Test public void testNegativeHitLimit() { - assertParse("select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", - "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1)", "foo:[0;1;-38]"); } @Test public void testRangeSearchHitPopulationOrdering() { - assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true}]range(foo, 0, 1);", "foo:[0;1;38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": false}]range(foo, 0, 1);", "foo:[0;1;-38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": true}]range(foo, 0, 1);", "foo:[0;1;-38]"); - assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": false}]range(foo, 0, 1);", "foo:[0;1;38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true}]range(foo, 0, 1)", + "foo:[0;1;38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": false}]range(foo, 0, 1)", + "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": true}]range(foo, 0, 1)", + "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": false}]range(foo, 0, 1)", + "foo:[0;1;38]"); boolean gotExceptionFromParse = false; try { - parse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true, \"descending\": false}]range(foo, 0, 1);"); + parse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true, \"descending\": false}]range(foo, 0, 1)"); } catch (IllegalArgumentException e) { assertTrue("Expected information about abuse of settings.", e.getMessage().contains("both ascending and descending ordering set")); @@ -875,23 +865,23 @@ public class YqlParserTestCase { @Test public void testOpenIntervals() { - assertParse("select * from sources * where range(title, 0.0, 500.0);", + assertParse("select * from sources * where range(title, 0.0, 500.0)", "title:[0.0;500.0]"); assertParse( - "select * from sources * where [{\"bounds\": \"open\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{\"bounds\": \"open\"}]range(title, 0.0, 500.0)", "title:<0.0;500.0>"); assertParse( - "select * from sources * where [{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0)", "title:<0.0;500.0]"); assertParse( - "select * from sources * where [{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0);", + "select * from sources * where [{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0)", "title:[0.0;500.0>"); } @Test public void testInheritedAnnotations() { { - QueryTree x = parse("select * from sources * where ([{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{\"ranked\": false}]\"c\");"); + QueryTree x = parse("select * from sources * where ([{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{\"ranked\": false}]\"c\")"); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(3, terms.size()); for (IndexedItem term : terms) { @@ -899,7 +889,7 @@ public class YqlParserTestCase { } } { - QueryTree x = parse("select * from sources * where [{\"ranked\": false}](foo contains \"a\" and bar contains \"b\");"); + QueryTree x = parse("select * from sources * where [{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")"); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(2, terms.size()); for (IndexedItem term : terms) { @@ -914,7 +904,7 @@ public class YqlParserTestCase { "([{\"ranked\": false}](foo contains \"a\" " + "and ([{\"ranked\": true}](bar contains \"b\" " + "or ([{\"ranked\": false}](foo contains \"c\" " + - "and foo contains ([{\"ranked\": true}]\"d\")))))));"; + "and foo contains ([{\"ranked\": true}]\"d\")))))))"; QueryTree x = parse(yqlQuery); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(4, terms.size()); @@ -945,7 +935,7 @@ public class YqlParserTestCase { ParserEnvironment parserEnvironment = new ParserEnvironment().setIndexFacts(indexFacts); YqlParser configuredParser = new YqlParser(parserEnvironment); QueryTree x = configuredParser.parse(new Parsable() - .setQuery("select * from sources * where title contains \"a\" and song contains \"b\";")); + .setQuery("select * from sources * where title contains \"a\" and song contains \"b\"")); List<IndexedItem> terms = QueryTree.getPositiveTerms(x); assertEquals(2, terms.size()); for (IndexedItem term : terms) { @@ -955,7 +945,7 @@ public class YqlParserTestCase { @Test public void testRegexp() { - QueryTree x = parse("select * from sources * where foo matches \"a b\";"); + QueryTree x = parse("select * from sources * where foo matches \"a b\""); Item root = x.getRoot(); assertSame(RegExpItem.class, root.getClass()); assertEquals("a b", ((RegExpItem) root).stringValue()); @@ -963,7 +953,7 @@ public class YqlParserTestCase { @Test public void testWordAlternatives() { - QueryTree x = parse("select * from sources * where foo contains alternatives({\"trees\": 1.0, \"tree\": 0.7});"); + QueryTree x = parse("select * from sources * where foo contains alternatives({\"trees\": 1.0, \"tree\": 0.7})"); Item root = x.getRoot(); assertSame(WordAlternativesItem.class, root.getClass()); WordAlternativesItem alternatives = (WordAlternativesItem) root; @@ -974,7 +964,7 @@ public class YqlParserTestCase { public void testWordAlternativesWithOrigin() { QueryTree q = parse("select * from sources * where foo contains" + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + - "alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); Item root = q.getRoot(); assertSame(WordAlternativesItem.class, root.getClass()); WordAlternativesItem alternatives = (WordAlternativesItem) root; @@ -989,7 +979,7 @@ public class YqlParserTestCase { @Test public void testWordAlternativesInPhrase() { QueryTree q = parse("select * from sources * where" + - " foo contains phrase(\"forest\", alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + " foo contains phrase(\"forest\", alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); Item root = q.getRoot(); assertSame(PhraseItem.class, root.getClass()); PhraseItem phrase = (PhraseItem) root; @@ -1011,7 +1001,7 @@ public class YqlParserTestCase { } { - Query query = new Query("search?yql=select%20*%20from%20testtype%20where%20title%20contains%20%22%5C%5C%22;"); + Query query = new Query("search?yql=select%20*%20from%20testtype%20where%20title%20contains%20%22%5C%5C%22"); // Cause parsing :-\ Chain<Searcher> searchChain = new Chain<>(new MinimalQueryInserter()); @@ -1030,7 +1020,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains uri(\"google.com\");"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains uri(\"google.com\")"); assertUrlQuery("urlfield.hostname", yql, false, true, true); } @@ -1041,7 +1031,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"endAnchor\": false }]uri(\"google.com\"));"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"endAnchor\": false }]uri(\"google.com\"))"); assertUrlQuery("urlfield.hostname", yql, false, false, true); } @@ -1052,7 +1042,7 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"startAnchor\": true }] uri(\"google.com\"));"); + yql.properties().set("yql", "select * from sources * where urlfield.hostname contains ([{\"startAnchor\": true }] uri(\"google.com\"))"); assertUrlQuery("urlfield.hostname", yql, true, true, true); } @@ -1063,19 +1053,19 @@ public class YqlParserTestCase { // YQL query Query yql = new Query(); - yql.properties().set("yql", "select * from sources * where urlfield contains uri(\"google.com\");"); + yql.properties().set("yql", "select * from sources * where urlfield contains uri(\"google.com\")"); assertUrlQuery("urlfield", yql, false, false, false); } @Test public void testReservedWordInSource() { - parse("select * from sources like where text contains \"test\";"); + parse("select * from sources like where text contains \"test\""); // success: parsed without exception } @Test public void testAndSegmenting() { - parse("select * from sources * where (default contains ([{\"stem\": false}]\"m\") AND default contains ([{\"origin\": {\"original\": \"m\'s\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"m\", \"s\"))) timeout 472;"); + parse("select * from sources * where (default contains ([{\"stem\": false}]\"m\") AND default contains ([{\"origin\": {\"original\": \"m\'s\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"m\", \"s\"))) timeout 472"); } private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e7e7101aff7..39223d6c031 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -87,6 +87,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -436,10 +437,12 @@ public class ApplicationController { // Validate new deployment spec thoroughly before storing it. controller.jobController().deploymentStatus(application.get()); - // Clear notifications for instances that are no longer declared - for (var name : existingInstances) - if ( ! declaredInstances.contains(name)) - controller.notificationsDb().removeNotifications(NotificationSource.from(application.get().id().instance(name))); + for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { + if ( ! notification.source().instance().map(declaredInstances::contains).orElse(false)) + controller.notificationsDb().removeNotifications(notification.source()); + if ( ! notification.source().zoneId().map(application.get().require(notification.source().instance().get()).deployments()::containsKey).orElse(false)) + controller.notificationsDb().removeNotifications(notification.source()); + } store(application); return application; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index c9cd7f41d2b..81daf0cbcfe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -9,14 +9,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import org.hamcrest.Matchers; import org.junit.Test; import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author mortent @@ -54,7 +53,7 @@ public class TenantRoleMaintainerTest { var roleService = tester.controller().serviceRegistry().roleService(); List<TenantName> tenantNames = ((MockRoleService) roleService).maintainedTenants(); - assertThat(tenantNames, Matchers.containsInAnyOrder(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant())); + assertTrue(tenantNames.containsAll(List.of(prodAppTenant2.application().id().tenant(), perfAppTenant1.application().id().tenant()))); } private long permanentDeployments(Instance instance) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java index c75e1c7832e..f0c54505bb1 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java @@ -44,7 +44,7 @@ public abstract class ContentPolicyTestEnvironment { protected PolicyTestFrame frame; private Set<Integer> nodes; protected static int[] bucketOneNodePreference = new int[]{ 3, 5, 7, 6, 8, 0, 9, 2, 1, 4 }; - protected boolean debug = true; + protected boolean debug = false; @Before public void setUp() throws Exception { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 97b06c802dd..c85d5675787 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -47,21 +47,28 @@ public class Flags { public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Default limit for when to apply termwise query evaluation", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag TLS_SIZE_FRACTION = defineDoubleFlag( + "tls-size-fraction", 0.07, + List.of("baldersheim"), "2021-12-20", "2022-02-01", + "Fraction of disk available for transaction log", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "LATENCY", - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_TASK_LIMIT = defineIntFlag( "feed-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-10-14", "2022-01-01", + List.of("geirst, baldersheim"), "2021-10-14", "2022-02-01", "The task limit used by the executors handling feed in proton", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -83,56 +90,56 @@ public class Flags { public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, - List.of("geirst, baldersheim"), "2021-10-21", "2022-01-01", + List.of("geirst, baldersheim"), "2021-10-21", "2022-02-01", "Max amount of memory holding updates to an attribute before we do a commit.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicationmanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the mbus request thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Should we skip the mbus reply thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2022-01-01", + List.of("vekterli"), "2020-12-02", "2022-02-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, - List.of("tokle", "bjormel"), "2020-12-02", "2022-01-01", + List.of("tokle", "bjormel"), "2020-12-02", "2022-02-01", "Whether the controller should hide shared routing layer endpoint", "Takes effect immediately", APPLICATION_ID @@ -140,35 +147,35 @@ public class Flags { public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "Optionally deliver async messages in own thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, - List.of("baldersheim"), "2020-12-02", "2022-01-01", + List.of("baldersheim"), "2020-12-02", "2022-02-01", "How much concurrency should be allowed for feed", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DISK_BLOAT_FACTOR = defineDoubleFlag( "disk-bloat-factor", 0.2, - List.of("baldersheim"), "2021-10-08", "2022-01-01", + List.of("baldersheim"), "2021-10-08", "2022-02-01", "Amount of bloat allowed before compacting file", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag DOCSTORE_COMPRESSION_LEVEL = defineIntFlag( "docstore-compression-level", 3, - List.of("baldersheim"), "2021-10-08", "2022-01-01", + List.of("baldersheim"), "2021-10-08", "2022-02-01", "Default compression level used for document store", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag NUM_DEPLOY_HELPER_THREADS = defineIntFlag( "num-model-builder-threads", -1, - List.of("balder"), "2021-09-09", "2022-01-01", + List.of("balder"), "2021-09-09", "2022-02-01", "Number of threads used for speeding up building of models.", "Takes effect on first (re)start of config server"); @@ -181,14 +188,14 @@ public class Flags { public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, - List.of("baldersheim"), "2021-09-25", "2022-01-01", + List.of("baldersheim"), "2021-09-25", "2022-02-01", "Will trigger a heap dump during if container shutdown times out", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, - List.of("baldersheim"), "2021-09-25", "2022-01-01", + List.of("baldersheim"), "2021-09-25", "2022-02-01", "Timeout for shutdown of a jdisc container", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -216,14 +223,14 @@ public class Flags { public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( "max-concurrent-merges-per-node", 128, - List.of("balder", "vekterli"), "2021-06-06", "2022-01-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( "max-merge-queue-size", 1024, - List.of("balder", "vekterli"), "2021-06-06", "2022-01-01", + List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); @@ -238,7 +245,7 @@ public class Flags { public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( "large-rank-expression-limit", 8192, - List.of("baldersheim"), "2021-06-09", "2022-01-01", + List.of("baldersheim"), "2021-06-09", "2022-02-01", "Limit for size of rank expressions distributed by filedistribution", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -252,14 +259,14 @@ public class Flags { public static final UnboundIntFlag METRICSPROXY_NUM_THREADS = defineIntFlag( "metricsproxy-num-threads", 2, - List.of("balder"), "2021-09-01", "2022-01-01", + List.of("balder"), "2021-09-01", "2022-02-01", "Number of threads for metrics proxy", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ENABLED_HORIZON_DASHBOARD = defineFeatureFlag( "enabled-horizon-dashboard", false, - List.of("olaa"), "2021-09-13", "2021-12-31", + List.of("olaa"), "2021-09-13", "2022-02-01", "Enable Horizon dashboard", "Takes effect immediately", TENANT_ID, CONSOLE_USER_EMAIL @@ -267,7 +274,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_ONPREM_TENANT_S3_ARCHIVE = defineFeatureFlag( "enable-onprem-tenant-s3-archive", false, - List.of("bjorncs"), "2021-09-14", "2021-12-31", + List.of("bjorncs"), "2021-09-14", "2022-02-01", "Enable tenant S3 buckets in cd/main. Must be set on controller cluster only.", "Takes effect immediately", ZONE_ID, TENANT_ID @@ -275,7 +282,7 @@ public class Flags { public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, - List.of("andreer"), "2021-09-23", "2021-12-21", + List.of("andreer"), "2021-09-23", "2022-02-01", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" ); @@ -289,7 +296,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_TENANT_DEVELOPER_ROLE = defineFeatureFlag( "enable-tenant-developer-role", false, - List.of("bjorncs"), "2021-09-23", "2021-12-31", + List.of("bjorncs"), "2021-09-23", "2022-02-01", "Enable tenant developer Athenz role in cd/main. Must be set on controller cluster only.", "Takes effect immediately", TENANT_ID @@ -297,7 +304,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_ROUTING_REUSE_PORT = defineFeatureFlag( "enable-routing-reuse-port", false, - List.of("mortent"), "2021-09-29", "2021-12-31", + List.of("mortent"), "2021-09-29", "2022-02-01", "Enable reuse port in routing configuration", "Takes effect on container restart", HOSTNAME @@ -305,7 +312,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_TENANT_OPERATOR_ROLE = defineFeatureFlag( "enable-tenant-operator-role", false, - List.of("bjorncs"), "2021-09-29", "2021-12-31", + List.of("bjorncs"), "2021-09-29", "2022-02-01", "Enable tenant specific operator roles in public systems. For controllers only.", "Takes effect on subsequent maintainer invocation", TENANT_ID @@ -367,7 +374,7 @@ public class Flags { public static final UnboundBooleanFlag USE_LEGACY_LB_SERVICES = defineFeatureFlag( "use-legacy-lb-services", false, - List.of("tokle"), "2021-11-22", "2021-12-31", + List.of("tokle"), "2021-11-22", "2022-02-01", "Whether to generate routing table based on legacy lb-services config", "Takes effect on container reboot", ZONE_ID, HOSTNAME); @@ -386,6 +393,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( + "fail-deployment-with-invalid-jvm-options", false, + List.of("hmusum"), "2021-12-20", "2022-01-20", + "Whether to fail deployments with invalid JVM options in services.xml", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index cfc7b09a934..32d8068dfcd 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -44,6 +44,11 @@ <artifactId>predicate-search-core</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/pom.xml b/linguistics/pom.xml index a09f2ecb031..d9ab942a0b8 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -22,6 +22,7 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -61,10 +62,6 @@ <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java b/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java new file mode 100644 index 00000000000..27c23d8d3e6 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/DefaultLanguageDetectorContextGenerator.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.ngram.NGramCharModel; +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.HashSet; +import java.util.Set; + +/** + * Avoids using the unnecessarily slow {@link NGramCharModel}. + * + * @author jonmv + */ +public class DefaultLanguageDetectorContextGenerator extends opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator { + + public DefaultLanguageDetectorContextGenerator(int minLength, int maxLength, CharSequenceNormalizer... normalizers) { + super(minLength, maxLength, normalizers); + } + + @Override + public String[] getContext(CharSequence document) { + int[] normalized = normalizer.normalize(document).codePoints().map(Character::toLowerCase).toArray(); + Set<String> grams = new HashSet<>(); + for (int i = 0; i < normalized.length; i++) + for (int j = minLength; j <= maxLength && i + j < normalized.length; j++) + grams.add(new String(normalized, i, j)); + + return grams.toArray(new String[grams.size()]); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java new file mode 100644 index 00000000000..fdca5355008 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.langdetect.LanguageDetectorContextGenerator; +import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer; +import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer; +import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer; +import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer; + +/** + * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 + * + * @author jonmv + */ +@SuppressWarnings("unused") // Loaded by black magic. +public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { + + @Override + public LanguageDetectorContextGenerator getContextGenerator() { + return new DefaultLanguageDetectorContextGenerator(1, 3, + EmojiCharSequenceNormalizer.getInstance(), + UrlCharSequenceNormalizer.getInstance(), + TwitterCharSequenceNormalizer.getInstance(), + NumberCharSequenceNormalizer.getInstance(), + ShrinkCharSequenceNormalizer.getInstance()); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java new file mode 100644 index 00000000000..d7a7d3a4744 --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpDetector.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detection; +import com.yahoo.language.detect.Detector; +import com.yahoo.language.detect.Hint; +import com.yahoo.language.simple.SimpleDetector; +import opennlp.tools.langdetect.LanguageDetectorConfig; +import opennlp.tools.langdetect.LanguageDetectorME; +import opennlp.tools.langdetect.LanguageDetectorModel; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Detects text language using patched OpenNLP, with fallback to {@link SimpleDetector} for undetected CJK input. + * + * @author jonmv + */ +class OpenNlpDetector implements Detector { + + private static final Object monitor = new Object(); + private static LanguageDetectorModel model; + + private final SimpleDetector simple = new SimpleDetector(); + private final Map<String, Language> languagesByISO3 = new HashMap<>(); + private final LanguageDetectorME detector; + private final LanguageDetectorConfig config; + + OpenNlpDetector() { + detector = new LanguageDetectorME(loadModel()); + config = new LanguageDetectorConfig(); + config.setMinDiff(0.02); + config.setChunkSize(32); + config.setMaxLength(256); + for (Locale locale : Locale.getAvailableLocales()) { + Language language = Language.fromLocale(locale); + if (language != null) + languagesByISO3.put(locale.getISO3Language(), language); + } + } + + private static LanguageDetectorModel loadModel() { + synchronized (monitor) { + if (model == null) { + try { + model = new LanguageDetectorModel(OpenNlpDetector.class.getResourceAsStream("/models/langdetect-183.bin")); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return model; + } + + @Override + public Detection detect(byte[] input, int offset, int length, Hint hint) { + Charset encoding = Charset.forName(simple.guessEncoding(input, offset, length)); + return new Detection(detectLanguage(new String(input, offset, length, encoding)), encoding.name(), false); + } + + @Override + public Detection detect(ByteBuffer input, Hint hint) { + if (input.hasArray()) + return detect(input.array(), input.arrayOffset() + input.position(), input.remaining(), hint); + + byte[] buffer = new byte[input.remaining()]; + input.get(buffer); + return detect(buffer, 0, buffer.length, hint); + } + + @Override + public Detection detect(String input, Hint hint) { + return new Detection(detectLanguage(input), UTF_8.name(), false); + } + + private Language detectLanguage(String input) { + var prediction = detector.probingPredictLanguages(input, config).getLanguages()[0]; + var result = prediction.getConfidence() > 0.02 ? languagesByISO3.get(prediction.getLang()) : null; + return result != null ? result : simple.guessLanguage(input.substring(0, Math.min(input.length(), 256))); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index a27e726cda8..c749679024a 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -7,36 +7,21 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; -import java.util.logging.Logger; -import java.util.logging.Level; +import opennlp.tools.langdetect.LanguageDetectorModel; /** - * Returns a linguistics implementation based on OpenNlp, - * and (optionally, default on) Optimaize for language detection. + * Returns a linguistics implementation based on OpenNlp. * * @author bratseth + * @author jonmv */ public class OpenNlpLinguistics extends SimpleLinguistics { - private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; - public OpenNlpLinguistics() { - this(true); - } - @Inject - public OpenNlpLinguistics(OpennlpLinguisticsConfig config) { - this(config.detector().enableOptimaize()); - } - - public OpenNlpLinguistics(boolean enableOptimaize) { - this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); - log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); - } - - private OpenNlpLinguistics(Detector detector) { - this.detector = detector; + public OpenNlpLinguistics() { + this.detector = new OpenNlpDetector(); } @Override diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java deleted file mode 100644 index 83947c795fb..00000000000 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.google.common.base.Optional; -import com.optimaize.langdetect.LanguageDetector; -import com.optimaize.langdetect.LanguageDetectorBuilder; -import com.optimaize.langdetect.i18n.LdLocale; -import com.optimaize.langdetect.ngram.NgramExtractors; -import com.optimaize.langdetect.profiles.LanguageProfile; -import com.optimaize.langdetect.profiles.LanguageProfileReader; -import com.optimaize.langdetect.text.CommonTextObjectFactories; -import com.optimaize.langdetect.text.TextObjectFactory; -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detection; -import com.yahoo.language.detect.Detector; -import com.yahoo.language.detect.Hint; -import com.yahoo.language.simple.SimpleDetector; -import com.yahoo.text.Utf8; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Locale; -import java.util.logging.Logger; -import java.util.logging.Level; - -/** - * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. - * - * @author bratseth - */ -public class OptimaizeDetector implements Detector { - - private static final Object initGuard = new Object(); - private static TextObjectFactory textObjectFactory = null; - private static LanguageDetector languageDetector = null; - private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); - - static private void initOptimaize() { - synchronized (initGuard) { - if ((textObjectFactory != null) && (languageDetector != null)) return; - - // origin: https://github.com/optimaize/language-detector - // load all languages: - List<LanguageProfile> languageProfiles; - try { - languageProfiles = new LanguageProfileReader().readAllBuiltIn(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - //build language detector: - languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) - .withProfiles(languageProfiles) - .build(); - - //create a text object factory - textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); - } - } - - private final SimpleDetector simpleDetector = new SimpleDetector(); - - public OptimaizeDetector() { - initOptimaize(); - } - - @Override - public Detection detect(byte[] input, int offset, int length, Hint hint) { - return new Detection(guessLanguage(input, offset, length), simpleDetector.guessEncoding(input), false); - } - - @Override - public Detection detect(ByteBuffer input, Hint hint) { - byte[] buf = new byte[input.remaining()]; - input.get(buf, 0, buf.length); - return detect(buf, 0, buf.length, hint); - } - - @Override - public Detection detect(String input, Hint hint) { - return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); - } - - private Language guessLanguage(byte[] buf, int offset, int length) { - return guessLanguage(Utf8.toString(buf, offset, length)); - } - - public Language guessLanguage(String input) { - if (input == null || input.length() == 0) return Language.UNKNOWN; - - Language result = simpleDetector.guessLanguage(input); - if (result != Language.UNKNOWN) return result; - - return guessLanguageUsingOptimaize(input); - } - - private static Language guessLanguageUsingOptimaize(String input) { - Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input)); - if ( ! result.isPresent()) return Language.UNKNOWN; - log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); - - return Language.fromLocale(new Locale(result.get().getLanguage())); - } - -} diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java new file mode 100644 index 00000000000..883319e2f8b --- /dev/null +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.regex.Pattern; + +/** + * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. + * + * @author jonmv + */ +public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { + + private static final Pattern URL_REGEX = + Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); + private static final Pattern MAIL_REGEX = + Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); + + private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); + + public static UrlCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + public CharSequence normalize(CharSequence text) { + String modified = URL_REGEX.matcher(text).replaceAll(" "); + return MAIL_REGEX.matcher(modified).replaceAll(" "); + } + +} diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 53b8ad7ad70..61d446cd8d0 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -130,10 +130,14 @@ public class SimpleDetector implements Detector { } public String guessEncoding(byte[] input) { + return guessEncoding(input, 0, input.length); + } + + public String guessEncoding(byte[] input, int offset, int length) { boolean isUtf8 = true; boolean hasHighs = false; scan: - for (int i = 0; i < input.length; i++) { + for (int i = offset; i < offset + length; i++) { final int l = isLeadingFor(input[i]); if (l < 0 || i + l >= input.length) { hasHighs = true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 3ca46dcc4f1..b10beb8c9af 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -2,8 +2,7 @@ package com.yahoo.language.simple; import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.Version; +import com.yahoo.component.AbstractComponent; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.CharacterClasses; @@ -16,7 +15,6 @@ import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import java.util.List; diff --git a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def b/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def deleted file mode 100644 index 361a8a5f50c..00000000000 --- a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=language.opennlp - -# Enable Optimaize language detector -detector.enableOptimaize bool default=true - diff --git a/linguistics/src/main/resources/models/langdetect-183.bin b/linguistics/src/main/resources/models/langdetect-183.bin Binary files differnew file mode 100644 index 00000000000..c3cde217050 --- /dev/null +++ b/linguistics/src/main/resources/models/langdetect-183.bin diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java new file mode 100644 index 00000000000..746ed10da1c --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpDetectorTestCase.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import com.yahoo.language.Language; +import com.yahoo.language.detect.Detector; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class OpenNlpDetectorTestCase { + + @Test + public void testDetection() { + Detector detector = new OpenNlpDetector(); + + assertLanguage(Language.UNKNOWN, + "", + detector); + + assertLanguage(Language.UNKNOWN, + "Hello!", + detector); + + // from https://en.wikipedia.org/wiki/Yahoo + assertLanguage(Language.ENGLISH, + "Yahoo became a public company via an initial public offering in April 1996 and its stock price rose 600% within two years.", + detector); + + // from https://de.wikipedia.org/wiki/Yahoo + assertLanguage(Language.GERMAN, + "1996 ging Yahoo mit 46 Angestellten an die Börse. 2009 arbeiteten insgesamt rund 13.500 Mitarbeiter für Yahoo.", + detector); + + // from https://fr.wikipedia.org/wiki/Yahoo + assertLanguage(Language.FRENCH, + "À l'origine, Yahoo! était uniquement un annuaire Web.", + detector); + + // Test fallback to SimpleDetector + assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input + "我能吞下玻璃而不伤身体。", + detector); + + // from https://zh.wikipedia.org/wiki/Yahoo + assertLanguage(Language.CHINESE_TRADITIONAL, + "Yahoo! Next是一个展示雅虎新技术、新产品的场所，目前在测试阶段。", + detector); + + // from https://th.wikipedia.org/wiki/Yahoo + assertLanguage(Language.THAI, + "เดือนกรกฎาคม 2012 Yahoo! ก็ได้ประธานเจ้าหน้าที่บริหารคนใหม่ \"มาริสสา เมเยอร์\" อดีตผู้บริหารจาก Google มาทำหน้าที่พลิกฟื้นบริษัท", + detector); + + // from https://ar.wikipedia.org/wiki/Yahoo + assertLanguage(Language.ARABIC, + "وفقًا لمزودي تحليلات الويب دائما كأليكسا وسميلارويب،وصل موقع ياهولأكثر من 7 مليارات مشاهدة شهريًا - حيث احتل المرتبة السادسة بين أكثر مواقع الويب زيارة على مستوى العالم في عام 2016.", + detector); + + // from https://ko.wikipedia.org/wiki/Yahoo + assertLanguage(Language.KOREAN, + "야후!의 전신인 디렉터리 사이트는 1994년 1월에 스탠퍼드 대학교 출신의 제리 양과 데이비드 파일로가 만들었으며, 회사는 1995년 3월 2일에 설립되었다.", + detector); + + // from https://ja.wikipedia.org/wiki/Yahoo + assertLanguage(Language.JAPANESE, + "日本では、ヤフー株式会社がYahoo!（後にベライゾンがアルタバに売却）とソフトバンクの合弁会社として1996年に設立した。", + detector); + + // from https://ru.wikipedia.org/wiki/Yahoo + assertLanguage(Language.RUSSIAN, + "7 февраля 2000 года Yahoo.com подвергся DDoS атаке и на несколько часов приостановил работу.", + detector); + + // from https://he.wikipedia.org/wiki/Yahoo + assertLanguage(Language.HEBREW, + "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום", + detector); + } + + private void assertLanguage(Language language, String input, Detector detector) { + assertEquals(language, detector.detect(input, null).getLanguage()); + } + +} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java deleted file mode 100644 index 20b5de3b165..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OptimaizeDetectorTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.opennlp; - -import com.yahoo.language.Language; -import com.yahoo.language.detect.Detector; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class OptimaizeDetectorTestCase { - - private static final Detector detector = new OptimaizeDetector(); - - @Test - public void testDetection() { - assertLanguage(Language.UNKNOWN, "Hello!"); - - // Test fallback to SimpleDetector - assertLanguage(Language.CHINESE_TRADITIONAL, // CHINESE_SIMPLIFIED input - "\u6211\u80FD\u541E\u4E0B\u73BB\u7483\u800C\u4E0D\u4F24\u8EAB\u4F53\u3002"); - - // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F - assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии"); - // https://he.wikipedia.org/wiki/Yahoo! - assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום"); - } - - private static void assertLanguage(Language language, String input) { - assertEquals(language, detector.detect(input, null).getLanguage()); - } - -} diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java new file mode 100644 index 00000000000..a8c637bc6ec --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class UrlCharSequenceNormalizerTest { + + @Test + public void testNormalization() { + String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo"; + assertEquals(" foo _bax a@b § ", + UrlCharSequenceNormalizer.getInstance().normalize(text)); + } + +} diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 1633d78668b..76f7e92ad43 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -147,11 +147,6 @@ <artifactId>jetty-http</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java index 457e27a5896..895bf344266 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collector; +import java.util.stream.Collectors; import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static java.util.Collections.unmodifiableSet; @@ -31,6 +32,7 @@ public class MetricsConsumers { // All metrics for each consumer. private final Map<ConsumerId, List<ConfiguredMetric>> consumerMetrics; + private final Map<ConsumerId, Map<MetricId, ConfiguredMetric>> configuredMetricByMetricByConsumer; // All consumers for each metric (more useful than the opposite map). private final Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric; @@ -42,6 +44,10 @@ public class MetricsConsumers { consumerMetrics = config.consumer().stream().collect( toUnmodifiableLinkedMap(consumer -> ConsumerId.toConsumerId(consumer.name()), consumer -> convert(consumer.metric()))); + configuredMetricByMetricByConsumer = new HashMap<>(); + consumerMetrics.forEach((consumer, configuredList) -> + configuredMetricByMetricByConsumer.put(consumer, + configuredList.stream().collect(Collectors.toMap(ConfiguredMetric::id, Function.identity())))); consumersByMetric = createConsumersByMetric(consumerMetrics); consumersByMetricByMetricId = new HashMap<>(); consumersByMetric.forEach((configuredMetric, consumers) -> { @@ -67,6 +73,10 @@ public class MetricsConsumers { return consumersByMetricByMetricId.get(id); } + public Map<MetricId, ConfiguredMetric> getMetricsForConsumer(ConsumerId consumerId) { + return configuredMetricByMetricByConsumer.get(consumerId); + } + public Set<ConsumerId> getAllConsumers() { return unmodifiableSet(consumerMetrics.keySet()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index de8d2c62880..f9ecaa29153 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -81,7 +81,16 @@ public class MetricsManager { * @return metrics for all matching services */ public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) { - MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime); + MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, null); + List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); + for (int i = 0; i < builderArray.length; i++) { + metricsPackets.add(builderArray[i].build()); + builderArray[i] = null; // Set null to be able to GC the builder when packet has been created + } + return metricsPackets; + } + public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + MetricsPacket.Builder [] builderArray = getMetricsBuildersAsArray(services, startTime, consumerId); List<MetricsPacket> metricsPackets = new ArrayList<>(builderArray.length); for (int i = 0; i < builderArray.length; i++) { metricsPackets.add(builderArray[i].build()); @@ -90,8 +99,8 @@ public class MetricsManager { return metricsPackets; } - public MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime) { - List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime); + private MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime, consumerId); return builders.toArray(new MetricsPacket.Builder[builders.size()]); } @@ -99,13 +108,13 @@ public class MetricsManager { * Returns the metrics for the given services, in mutable state for further processing. * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary. */ - public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) { + public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime, ConsumerId consumerId) { if (services.isEmpty()) return Collections.emptyList(); log.log(FINE, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); - List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services); + List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services, consumerId); log.log(FINE, () -> "Got " + result.size() + " metrics packets for vespa services."); purgeStaleMetrics(); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index 93f8ec0440b..ebb5d2fe8fb 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -15,6 +15,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.service.MetricsParser; import ai.vespa.metricsproxy.service.VespaService; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -65,7 +66,7 @@ public class VespaMetrics { * @param services the services to get metrics for * @return a list of metrics packet builders (to allow modification by the caller) */ - public List<MetricsPacket.Builder> getMetrics(List<VespaService> services) { + public List<MetricsPacket.Builder> getMetrics(List<VespaService> services, ConsumerId consumerId) { List<MetricsPacket.Builder> metricsPackets = new ArrayList<>(); for (VespaService service : services) { @@ -74,7 +75,9 @@ public class VespaMetrics { systemCheck.ifPresent(metricsPackets::add); MetricAggregator aggregator = new MetricAggregator(service.getDimensions()); - GetServiceMetricsConsumer metricsConsumer = new GetServiceMetricsConsumer(metricsConsumers, aggregator); + MetricsParser.Consumer metricsConsumer = (consumerId != null) + ? new GetServiceMetricsConsumer(metricsConsumers, aggregator, consumerId) + : new GetServiceMetricsConsumerForAll(metricsConsumers, aggregator); service.consumeMetrics(metricsConsumer); if (! aggregator.getAggregated().isEmpty()) { @@ -115,24 +118,14 @@ public class VespaMetrics { * In order to include a metric, it must exist in the given map of metric to consumers. * Each returned metric will contain a collection of consumers that it should be routed to. */ - private static class GetServiceMetricsConsumer implements MetricsParser.Consumer { - private final MetricAggregator aggregator; - private final MetricsConsumers metricsConsumers; - GetServiceMetricsConsumer(MetricsConsumers metricsConsumers, MetricAggregator aggregator) { - this.metricsConsumers = metricsConsumers; + private static abstract class GetServiceMetricsConsumerBase implements MetricsParser.Consumer { + protected final MetricAggregator aggregator; + + GetServiceMetricsConsumerBase(MetricAggregator aggregator) { this.aggregator = aggregator; } - @Override - public void consume(Metric candidate) { - Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(candidate.getName()); - if (consumersByMetric != null) { - consumersByMetric.keySet().forEach( - configuredMetric -> aggregator.aggregate( - metricWithConfigProperties(candidate, configuredMetric, consumersByMetric.get(configuredMetric)))); - } - } - private static Metric metricWithConfigProperties(Metric candidate, + protected static Metric metricWithConfigProperties(Metric candidate, ConfiguredMetric configuredMetric, Set<ConsumerId> consumers) { Metric metric = candidate.clone(); @@ -153,8 +146,46 @@ public class VespaMetrics { } private static Set<ConsumerId> extractConsumers(Set<ConsumerId> configuredConsumers) { - if (configuredConsumers != null) return configuredConsumers; - return Set.of(); + return (configuredConsumers != null) ? configuredConsumers : Set.of(); + } + } + + private static class GetServiceMetricsConsumer extends GetServiceMetricsConsumerBase { + private final Map<MetricId, ConfiguredMetric> configuredMetrics; + private final Set<ConsumerId> consumerId; + + GetServiceMetricsConsumer(MetricsConsumers metricsConsumers, MetricAggregator aggregator, ConsumerId consumerId) { + super(aggregator); + this.consumerId = Set.of(consumerId); + this.configuredMetrics = metricsConsumers.getMetricsForConsumer(consumerId); + } + + @Override + public void consume(Metric candidate) { + ConfiguredMetric configuredMetric = configuredMetrics.get(candidate.getName()); + if (configuredMetric != null) { + aggregator.aggregate( + metricWithConfigProperties(candidate, configuredMetric, consumerId)); + } + } + } + + private static class GetServiceMetricsConsumerForAll extends GetServiceMetricsConsumerBase { + private final MetricsConsumers metricsConsumers; + + GetServiceMetricsConsumerForAll(MetricsConsumers metricsConsumers, MetricAggregator aggregator) { + super(aggregator); + this.metricsConsumers = metricsConsumers; + } + + @Override + public void consume(Metric candidate) { + Map<ConfiguredMetric, Set<ConsumerId>> consumersByMetric = metricsConsumers.getConsumersByMetric(candidate.getName()); + if (consumersByMetric != null) { + consumersByMetric.keySet().forEach( + configuredMetric -> aggregator.aggregate( + metricWithConfigProperties(candidate, configuredMetric, consumersByMetric.get(configuredMetric)))); + } } } @@ -197,8 +228,8 @@ public class VespaMetrics { return definitions == null ? Collections.emptyList() : definitions; } - private static void setMetaInfo(MetricsPacket.Builder builder, long timestamp) { - builder.timestamp(timestamp) + private static void setMetaInfo(MetricsPacket.Builder builder, Instant timestamp) { + builder.timestamp(timestamp.getEpochSecond()) .statusCode(0) .statusMessage("Data collected successfully"); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 8f7f1c8a779..df670addcd8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -41,7 +41,7 @@ public class ValuesFetcher { public List<MetricsPacket> fetch(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); - return fetchAllMetrics() + return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now(), consumer) .stream() .filter(metricsPacket -> metricsPacket.consumers().contains(consumer)) .collect(Collectors.toList()); @@ -50,7 +50,7 @@ public class ValuesFetcher { public MetricsPacket.Builder [] fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); - List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now()) + List<MetricsPacket.Builder> builders = metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now(), consumer) .stream() .filter(builder -> builder.hasConsumer(consumer)) .collect(Collectors.toList()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java index 9265e8ef1e5..d12685be97d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java @@ -5,6 +5,7 @@ import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricId; +import java.time.Instant; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -14,7 +15,7 @@ import java.util.Set; */ public class Metric { - private final long time; + private final Instant time; private final Number value; private final String description; private MetricId name; @@ -28,7 +29,7 @@ public class Metric { * @param value The numeric value * @param time The timestamp of this metric in seconds */ - public Metric(MetricId name, Number value, long time, Map<DimensionId, String> dimensions, String description) { + public Metric(MetricId name, Number value, Instant time, Map<DimensionId, String> dimensions, String description) { this.time = time; this.value = value; this.name = name; @@ -37,11 +38,15 @@ public class Metric { } public Metric(MetricId name, Number value, long timestamp) { + this(name, value, Instant.ofEpochSecond(timestamp), Map.of(), ""); + } + + public Metric(MetricId name, Number value, Instant timestamp) { this(name, value, timestamp, Map.of(), ""); } public Metric(MetricId name, Number value) { - this(name, value, System.currentTimeMillis() / 1000); + this(name, value, Instant.now()); } public void setDimensions(Map<DimensionId, String> dimensions) { @@ -86,7 +91,7 @@ public class Metric { /** * @return The UTC timestamp for when this metric was collected */ - public long getTimeStamp() { + public Instant getTimeStamp() { return this.time; } @@ -112,7 +117,7 @@ public class Metric { return name.equals(rhs.name) && description.equals(rhs.description) && value.equals(rhs.value) - && (time == rhs.time) + && time.equals(rhs.time) && Objects.equals(dimensions, rhs.dimensions) && Objects.equals(consumers, rhs.consumers); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java index 8450d9f6be7..ebc206c9245 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -13,14 +14,14 @@ import java.util.List; public class Metrics { private final List<Metric> metrics = new ArrayList<>(); - private long timestamp; + private Instant timestamp; private boolean isFrozen = false; public Metrics() { - this(System.currentTimeMillis() / 1000L); + this(Instant.now()); } - public Metrics(long timestamp) { + public Metrics(Instant timestamp) { this.timestamp = timestamp; } @@ -28,7 +29,7 @@ public class Metrics { if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!"); } - public long getTimeStamp() { + public Instant getTimeStamp() { return this.timestamp; } @@ -37,7 +38,7 @@ public class Metrics { * * @param timestamp IN UTC seconds resolution */ - public void setTimeStamp(long timestamp) { + public void setTimeStamp(Instant timestamp) { ensureNotFrozen(); this.timestamp = timestamp; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 54ffcd4ae0f..35761ebcf18 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -110,7 +110,7 @@ public class MetricsPacket { public Builder putMetrics(Collection<Metric> extraMetrics) { if (extraMetrics != null) - extraMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue().doubleValue())); + extraMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue())); return this; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 65c9bb1ff76..87bed1c79e3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -77,15 +77,14 @@ public abstract class HttpMetricFetcher { } void handleException(Exception e, Object data, int timesFetched) { - logMessage("Unable to parse json '" + data + "' for service '" + service + "': " + - Exceptions.toMessageString(e), timesFetched); + logMessage("Unable to parse json '" + data + "' for service '" + service + "': ", e, timesFetched); } - private void logMessage(String message, int timesFetched) { + private void logMessage(String message, Exception e, int timesFetched) { if (service.isAlive() && timesFetched > 5) { - log.log(Level.INFO, message); + log.log(Level.INFO, message, e); } else { - log.log(Level.FINE, message); + log.log(Level.FINE, message, e); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java index f9443c46bad..bb21485e0e7 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/MetricsParser.java @@ -6,14 +6,16 @@ import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricId; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; @@ -29,7 +31,7 @@ public class MetricsParser { private static final ObjectMapper jsonMapper = new ObjectMapper(); - static void parse(String data, Consumer consumer) throws IOException { + public static void parse(String data, Consumer consumer) throws IOException { parse(jsonMapper.createParser(data), consumer); } @@ -53,21 +55,17 @@ public class MetricsParser { } } } - private static long secondsSince1970UTC() { - return System.currentTimeMillis() / 1000L; - } - static private long parseSnapshot(JsonParser parser) throws IOException { + static private Instant parseSnapshot(JsonParser parser) throws IOException { if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'snapshot' object, got " + parser.currentToken()); } - long timestamp = secondsSince1970UTC(); + Instant timestamp = Instant.now(); for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("to")) { - timestamp = parser.getLongValue(); - long now = System.currentTimeMillis() / 1000; - timestamp = Metric.adjustTime(timestamp, now); + timestamp = Instant.ofEpochSecond(parser.getLongValue()); + timestamp = Instant.ofEpochSecond(Metric.adjustTime(timestamp.getEpochSecond(), Instant.now().getEpochSecond())); } else { if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { parser.skipChildren(); @@ -77,19 +75,14 @@ public class MetricsParser { return timestamp; } - static private void parseValues(JsonParser parser, long timestamp, Consumer consumer) throws IOException { + static private void parseMetricValues(JsonParser parser, Instant timestamp, Consumer consumer) throws IOException { if (parser.getCurrentToken() != JsonToken.START_ARRAY) { throw new IOException("Expected start of 'metrics:values' array, got " + parser.currentToken()); } - Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); + Map<Long, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); while (parser.nextToken() == JsonToken.START_OBJECT) { - // read everything from this START_OBJECT to the matching END_OBJECT - // and return it as a tree model ObjectNode - JsonNode value = jsonMapper.readTree(parser); - handleValue(value, timestamp, consumer, uniqueDimensions); - - // do whatever you need to do with this object + handleValue(parser, timestamp, consumer, uniqueDimensions); } } @@ -97,14 +90,14 @@ public class MetricsParser { if (parser.getCurrentToken() != JsonToken.START_OBJECT) { throw new IOException("Expected start of 'metrics' object, got " + parser.currentToken()); } - long timestamp = System.currentTimeMillis() / 1000L; + Instant timestamp = Instant.now(); for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { String fieldName = parser.getCurrentName(); JsonToken token = parser.nextToken(); if (fieldName.equals("snapshot")) { timestamp = parseSnapshot(parser); } else if (fieldName.equals("values")) { - parseValues(parser, timestamp, consumer); + parseMetricValues(parser, timestamp, consumer); } else { if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { parser.skipChildren(); @@ -113,50 +106,69 @@ public class MetricsParser { } } - static private void handleValue(JsonNode metric, long timestamp, Consumer consumer, - Map<String, Map<DimensionId, String>> uniqueDimensions) { - String name = metric.get("name").textValue(); - String description = ""; - - if (metric.has("description")) { - description = metric.get("description").textValue(); + private static Map<DimensionId, String> parseDimensions(JsonParser parser, + Map<Long, Map<DimensionId, String>> uniqueDimensions) throws IOException { + List<Map.Entry<String, String>> dims = new ArrayList<>(); + int keyHash = 0; + int valueHash = 0; + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (token == JsonToken.VALUE_STRING){ + String value = parser.getValueAsString(); + dims.add(Map.entry(fieldName, value)); + keyHash ^= fieldName.hashCode(); + valueHash ^= value.hashCode(); + } else if (token == JsonToken.VALUE_NULL) { + // TODO Should log a warning if this happens + } else { + throw new IllegalArgumentException("Dimension '" + fieldName + "' must be a string"); + } } - - Map<DimensionId, String> dim = Map.of(); - if (metric.has("dimensions")) { - JsonNode dimensions = metric.get("dimensions"); - StringBuilder sb = new StringBuilder(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).asText(); - sb.append(toDimensionId(k)).append(v); + Long uniqueKey = (((long) keyHash) << 32) | (valueHash & 0xffffffffL); + return uniqueDimensions.computeIfAbsent(uniqueKey, key -> dims.stream().collect(Collectors.toUnmodifiableMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue))); + } + private static List<Map.Entry<String, Number>> parseValues(String prefix, JsonParser parser) throws IOException { + List<Map.Entry<String, Number>> metrics = new ArrayList<>(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + String metricName = prefix + fieldName; + if (token == JsonToken.VALUE_NUMBER_INT) { + metrics.add(Map.entry(metricName, parser.getLongValue())); + } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { + metrics.add(Map.entry(metricName, parser.getValueAsDouble())); + } else { + throw new IllegalArgumentException("Value for aggregator '" + fieldName + "' is not a number"); } - if ( ! uniqueDimensions.containsKey(sb.toString())) { - dim = new HashMap<>(); - for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.get(k).textValue(); - dim.put(toDimensionId(k), v); + } + return metrics; + } + static private void handleValue(JsonParser parser, Instant timestamp, Consumer consumer, + Map<Long, Map<DimensionId, String>> uniqueDimensions) throws IOException { + String name = ""; + String description = ""; + Map<DimensionId, String> dim = Map.of(); + List<Map.Entry<String, Number>> values = List.of(); + for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parser.getCurrentName(); + JsonToken token = parser.nextToken(); + if (fieldName.equals("name")) { + name = parser.getText(); + } else if (fieldName.equals("description")) { + description = parser.getText(); + } else if (fieldName.equals("dimensions")) { + dim = parseDimensions(parser, uniqueDimensions); + } else if (fieldName.equals("values")) { + values = parseValues(name+".", parser); + } else { + if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) { + parser.skipChildren(); } - uniqueDimensions.put(sb.toString(), Map.copyOf(dim)); } - dim = uniqueDimensions.get(sb.toString()); } - - JsonNode aggregates = metric.get("values"); - String prefix = name + "."; - for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) { - String aggregator = (String) it.next(); - JsonNode aggregatorValue = aggregates.get(aggregator); - if (aggregatorValue == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing"); - } - Number value = aggregatorValue.numberValue(); - if (value == null) { - throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number"); - } - String metricName = prefix + aggregator; - consumer.consume(new Metric(MetricId.toMetricId(metricName), value, timestamp, dim, description)); + for (Map.Entry<String, Number> value : values) { + consumer.consume(new Metric(MetricId.toMetricId(value.getKey()), value.getValue(), timestamp, dim, description)); } } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index f375fd1586f..5ca9e6fd950 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -28,12 +28,8 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { } } - void createMetrics(String data, MetricsParser.Consumer consumer, int fetchCount) { - try { - MetricsParser.parse(data, consumer); - } catch (Exception e) { - handleException(e, data, fetchCount); - } + void createMetrics(String data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { + MetricsParser.parse(data, consumer); } private void createMetrics(InputStream data, MetricsParser.Consumer consumer, int fetchCount) throws IOException { try { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 22eb2844d61..d159fdd3dab 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -161,7 +161,7 @@ public class SystemPoller { log.log(Level.FINE, () -> "Monitoring system metrics for " + services.size() + " services"); boolean someAlive = services.stream().anyMatch(VespaService::isAlive); - lastTotalCpuJiffies = updateMetrics(lastTotalCpuJiffies, interval.getSeconds(), jiffiesInterface, services, lastCpuJiffiesMetrics); + lastTotalCpuJiffies = updateMetrics(lastTotalCpuJiffies, startTime, jiffiesInterface, services, lastCpuJiffiesMetrics); // If none of the services were alive, reschedule in a short time if (!someAlive) { @@ -171,7 +171,7 @@ public class SystemPoller { } } - static JiffiesAndCpus updateMetrics(JiffiesAndCpus prevTotalJiffies, long timeStamp, GetJiffies getJiffies, + static JiffiesAndCpus updateMetrics(JiffiesAndCpus prevTotalJiffies, Instant timeStamp, GetJiffies getJiffies, List<VespaService> services, Map<VespaService, Long> lastCpuJiffiesMetrics) { Map<VespaService, Long> currentServiceJiffies = new HashMap<>(); for (VespaService s : services) { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java index c02e76320f2..9e5a24ccd3c 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java @@ -34,11 +34,8 @@ import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -72,40 +69,40 @@ public class MetricsManagerTest { getMetricsConsumers(),getApplicationDimensions(), getNodeDimensions()); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(1)); + assertEquals(1, packets.size()); assertTrue(packets.get(0).metrics().isEmpty()); - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is(DownService.NAME)); - assertThat(packets.get(0).dimensions().get(toDimensionId("global")), is("value")); + assertEquals(DownService.NAME, packets.get(0).dimensions().get(toDimensionId("instance"))); + assertEquals("value", packets.get(0).dimensions().get(toDimensionId("global"))); } @Test public void each_service_gets_separate_metrics_packets() { List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); + assertEquals(2, packets.size()); - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is("dummy0")); - assertThat(packets.get(0).metrics().get(toMetricId("c.test")), is(1.0)); - assertThat(packets.get(0).metrics().get(toMetricId("val")), is(1.05)); + assertEquals("dummy0", packets.get(0).dimensions().get(toDimensionId("instance"))); + assertEquals(1, packets.get(0).metrics().get(toMetricId("c.test"))); + assertEquals(1.05, packets.get(0).metrics().get(toMetricId("val"))); - assertThat(packets.get(1).dimensions().get(toDimensionId("instance")), is("dummy1")); - assertThat(packets.get(1).metrics().get(toMetricId("c.test")), is(6.0)); - assertThat(packets.get(1).metrics().get(toMetricId("val")), is(2.35)); + assertEquals("dummy1", packets.get(1).dimensions().get(toDimensionId("instance"))); + assertEquals(6, packets.get(1).metrics().get(toMetricId("c.test"))); + assertEquals(2.35, packets.get(1).metrics().get(toMetricId("val"))); } @Test public void verify_expected_output_from_getMetricsById() { String dummy0Metrics = metricsManager.getMetricsByConfigId(SERVICE_0_ID); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050")); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1")); + assertTrue(dummy0Metrics.contains("'dummy.id.0'.val=1.050")); + assertTrue(dummy0Metrics.contains("'dummy.id.0'.c_test=1")); String dummy1Metrics = metricsManager.getMetricsByConfigId(SERVICE_1_ID); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350")); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6")); + assertTrue(dummy1Metrics.contains("'dummy.id.1'.val=2.350")); + assertTrue(dummy1Metrics.contains("'dummy.id.1'.c_test=6")); } @Test public void getServices_returns_service_types() { - assertThat(metricsManager.getAllVespaServices(), is("dummy")); + assertEquals("dummy", metricsManager.getAllVespaServices()); } @Test @@ -134,8 +131,8 @@ public class MetricsManagerTest { assertEquals(3, packets.size()); MetricsPacket systemPacket = packets.get(0); // system metrics are added before other metrics - assertThat(systemPacket.metrics().get(toMetricId("cpu")), is(1.0)); - assertThat(systemPacket.dimensions().get(toDimensionId("metrictype")), is("system")); + assertEquals(1, systemPacket.metrics().get(toMetricId("cpu"))); + assertEquals("system", systemPacket.dimensions().get(toDimensionId("metrictype"))); service0.setSystemMetrics(oldSystemMetrics); } @@ -148,7 +145,7 @@ public class MetricsManagerTest { .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(3)); + assertEquals(3, packets.size()); } @Test @@ -158,7 +155,7 @@ public class MetricsManagerTest { .putMetrics(ImmutableList.of(new Metric(toMetricId("not-whitelisted"), 0))))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); + assertEquals(2, packets.size()); } @Test @@ -185,7 +182,7 @@ public class MetricsManagerTest { List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); for (MetricsPacket packet : packets) { - assertThat(packet.dimensions().get(ROLE_DIMENSION), is("role from extraMetrics")); + assertEquals("role from extraMetrics", packet.dimensions().get(ROLE_DIMENSION)); } } @@ -197,9 +194,9 @@ public class MetricsManagerTest { .putDimension(METRIC_TYPE_DIMENSION_ID, "from extraMetrics"))); List<MetricsPacket> packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("from extraMetrics")); + assertEquals("standard", packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID)); + assertEquals("standard", packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID)); + assertEquals("from extraMetrics", packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID)); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java index 1cd0281b7cd..46cc9b7b7fd 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java @@ -9,15 +9,15 @@ import org.junit.Test; import java.util.HashMap; import java.util.Map; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Unknowm */ public class MetricsTest { - + private static final double EPSILON = 0.00000000001; @Test public void testIterator() { Metrics m = new Metrics(); @@ -33,36 +33,36 @@ public class MetricsTest { for (Metric metric: m.list()) { String k = metric.getName().id; - assertThat(map.containsKey(k), is(false)); + assertFalse(map.containsKey(k)); map.put(k, metric.getValue()); } - assertThat(map.get("a").intValue(), is(1)); - assertThat(map.get("b").doubleValue(), is(2.5)); + assertEquals(1, map.get("a").intValue()); + assertEquals(2.5, map.get("b").doubleValue(), EPSILON); } @Test public void testBasicMetric() { Metrics m = new Metrics(); m.add(new Metric(toMetricId("count"), 1, System.currentTimeMillis() / 1000)); - assertThat(m.list().size(), is(1)); - assertThat(m.list().get(0).getName(), is(toMetricId("count"))); + assertEquals(1, m.list().size()); + assertEquals(toMetricId("count"), m.list().get(0).getName()); } @Test public void testHealthMetric() { HealthMetric m = HealthMetric.get(null, null); - assertThat(m.isOk(), is(false)); + assertFalse(m.isOk()); m = HealthMetric.get("up", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); + assertTrue(m.isOk()); + assertEquals("test message", m.getMessage()); m = HealthMetric.get("ok", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); + assertTrue(m.isOk()); + assertEquals("test message", m.getMessage()); m = HealthMetric.get("bad", "test message"); - assertThat(m.isOk(), is(false)); - assertThat(m.getStatus(), is(StatusCode.UNKNOWN)); + assertFalse(m.isOk()); + assertEquals(StatusCode.UNKNOWN, m.getStatus()); } @Test @@ -70,30 +70,30 @@ public class MetricsTest { MetricsFormatter formatter = new MetricsFormatter(false, false); VespaService service = new DummyService(0, "config.id"); String data = formatter.format(service, "key", 1); - assertThat(data, is("'config.id'.key=1")); + assertEquals("'config.id'.key=1", data); formatter = new MetricsFormatter(true, false); data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.'config.id'.key=1")); + assertEquals("dummy.'config.id'.key=1", data); formatter = new MetricsFormatter(true, true); data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.config.'id'.key=1")); + assertEquals("dummy.config.'id'.key=1", data); formatter = new MetricsFormatter(false, true); data = formatter.format(service, "key", 1); - assertThat(data, is("config.'id'.key=1")); + assertEquals("config.'id'.key=1", data); } @Test public void testTimeAdjustment() { - assertThat(Metric.adjustTime(0L, 0L), is(0L)); - assertThat(Metric.adjustTime(59L, 59L), is(59L)); - assertThat(Metric.adjustTime(60L, 60L), is(60L)); - assertThat(Metric.adjustTime(59L, 60L), is(60L)); - assertThat(Metric.adjustTime(60L, 59L), is(60L)); - assertThat(Metric.adjustTime(59L, 61L), is(59L)); + assertEquals(0L, Metric.adjustTime(0L, 0L)); + assertEquals(59L, Metric.adjustTime(59L, 59L)); + assertEquals(60L, Metric.adjustTime(60L, 60L)); + assertEquals(60L, Metric.adjustTime(59L, 60L)); + assertEquals(60L, Metric.adjustTime(60L, 59L)); + assertEquals(59L, Metric.adjustTime(59L, 61L)); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java index 268eab9f5d3..98fa337fa78 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java @@ -16,9 +16,10 @@ import java.util.List; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author jobergum @@ -40,21 +41,21 @@ public class RpcHealthMetricsTest { mockHttpServer.setResponse(HEALTH_OK_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat(services.size(), is(1)); + assertEquals(1, services.size()); VespaService qrserver = services.get(0); HealthMetric h = qrserver.getHealth(); assertNotNull("Health metric should never be null", h); - assertThat("Status failed, reason = " + h.getMessage(), h.isOk(), is(true)); - assertThat(h.getMessage(), is("WORKING")); + assertTrue("Status failed, reason = " + h.getMessage(), h.isOk()); + assertEquals("WORKING", h.getMessage()); mockHttpServer.setResponse(HEALTH_FAILED_RESPONSE); h = qrserver.getHealth(); assertNotNull("Health metric should never be null", h); - assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false)); - assertThat(h.getMessage(), is("SOMETHING FAILED")); + assertFalse("Status should be failed" + h.getMessage(), h.isOk()); + assertEquals("SOMETHING FAILED", h.getMessage()); String jsonRPCMessage = getHealthMetrics(tester, qrserver.getMonitoringName().id); - assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE)); + assertEquals(WANTED_RPC_RESPONSE, jsonRPCMessage); } } @@ -62,7 +63,7 @@ public class RpcHealthMetricsTest { public void non_existent_service_name_returns_an_error_message() { try (IntegrationTester tester = new IntegrationTester()) { String jsonRPCMessage = getHealthMetrics(tester, "non-existing service"); - assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'")); + assertEquals("105: No service with name 'non-existing service'", jsonRPCMessage); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index 8a489ac70c7..beb0d3fce60 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -28,12 +28,9 @@ import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_2_CONFIG_ID; import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -72,8 +69,8 @@ public class RpcMetricsTest { // Verify that application is used as serviceId, and that metric exists. JsonNode extraMetrics = findExtraMetricsObject(allServicesResponse); - assertThat(extraMetrics.get("metrics").get("foo.count").intValue(), is(3)); - assertThat(extraMetrics.get("dimensions").get("role").textValue(), is("extra-role")); + assertEquals(3, extraMetrics.get("metrics").get("foo.count").intValue()); + assertEquals("extra-role", extraMetrics.get("dimensions").get("role").textValue()); } } } @@ -101,13 +98,13 @@ public class RpcMetricsTest { tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1)); + assertEquals("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, 1, services.size()); VespaService qrserver = services.get(0); - assertThat(qrserver.getMonitoringName().id, is(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver")); + assertEquals(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver", qrserver.getMonitoringName().id); Metrics metrics = qrserver.getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); + assertEquals("Fetched number of metrics is not correct", 2, metrics.size()); Metric m = getMetric("foo.count", metrics); assertNotNull("Did not find expected metric with name 'foo.count'", m); Metric m2 = getMetric("bar.count", metrics); @@ -117,16 +114,16 @@ public class RpcMetricsTest { verifyMetricsFromRpcRequest(qrserver, rpcClient); services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); + assertEquals("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, 1, services.size()); VespaService storageService = services.get(0); verfiyMetricsFromServiceObject(storageService); String metricsById = getMetricsById(storageService.getConfigId(), rpcClient); - assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); + assertEquals("'storage.cluster.storage.storage.0'.foo_count=1 ", metricsById); String jsonResponse = getMetricsForYamas("non-existing", rpcClient).trim(); - assertThat(jsonResponse, is("105: No service with name 'non-existing'")); + assertEquals("105: No service with name 'non-existing'", jsonResponse); verifyMetricsFromRpcRequestForAllServices(rpcClient); @@ -145,21 +142,21 @@ public class RpcMetricsTest { private static void verifyMetricsFromRpcRequest(VespaService service, RpcClient client) throws IOException { String jsonResponse = getMetricsForYamas(service.getMonitoringName().id, client).trim(); ArrayNode metrics = (ArrayNode) jsonMapper.readTree(jsonResponse).get("metrics"); - assertThat("Expected 3 metric messages", metrics.size(), is(3)); + assertEquals("Expected 3 metric messages", 3, metrics.size()); for (int i = 0; i < metrics.size() - 1; i++) { // The last "metric message" contains only status code/message JsonNode jsonObject = metrics.get(i); assertFalse(jsonObject.has("status_code")); assertFalse(jsonObject.has("status_msg")); assertEquals("bar", jsonObject.get("dimensions").get("foo").textValue()); - assertThat(jsonObject.get("dimensions").get("bar").textValue(), is("foo")); - assertThat(jsonObject.get("dimensions").get("serviceDim").textValue(), is("serviceDimValue")); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").size(), is(1)); + assertEquals("foo", jsonObject.get("dimensions").get("bar").textValue()); + assertEquals("serviceDimValue", jsonObject.get("dimensions").get("serviceDim").textValue()); + assertEquals(1, jsonObject.get("routing").get("yamas").get("namespaces").size()); if (jsonObject.get("metrics").has("foo_count")) { - assertThat(jsonObject.get("metrics").get("foo_count").intValue(), is(1)); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(vespaMetricsConsumerId.id)); + assertEquals(1, jsonObject.get("metrics").get("foo_count").intValue()); + assertEquals(vespaMetricsConsumerId.id, jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue()); } else { - assertThat(jsonObject.get("metrics").get("foo.count").intValue(), is(1)); - assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(CUSTOM_CONSUMER_ID.id)); + assertEquals(1, jsonObject.get("metrics").get("foo.count").intValue()); + assertEquals(CUSTOM_CONSUMER_ID.id, jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue()); } } @@ -168,21 +165,21 @@ public class RpcMetricsTest { private void verfiyMetricsFromServiceObject(VespaService service) { Metrics storageMetrics = service.getMetrics(); - assertThat(storageMetrics.size(), is(2)); + assertEquals(2, storageMetrics.size()); Metric foo = getMetric("foo.count", storageMetrics); assertNotNull("Did not find expected metric with name 'foo.count'", foo); - assertThat("Expected 2 dimensions for metric foo", foo.getDimensions().size(), is(2)); - assertThat("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo")), is(true)); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().get(toDimensionId("foo")), is("bar")); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar")), is(true)); - assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); + assertEquals("Expected 2 dimensions for metric foo", 2, foo.getDimensions().size()); + assertTrue("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo"))); + assertEquals("Metric foo did not contain correct dimension", "bar", foo.getDimensions().get(toDimensionId("foo"))); + assertTrue("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar"))); + assertEquals("Metric foo did not contain correct dimension for key = bar", "foo", foo.getDimensions().get(toDimensionId("bar"))); } private void verifyMetricsFromRpcRequestForAllServices(RpcClient client) throws IOException { // Verify that metrics for all services can be retrieved in one request. String allServicesResponse = getMetricsForYamas(ALL_SERVICES, client).trim(); ArrayNode allServicesMetrics = (ArrayNode) jsonMapper.readTree(allServicesResponse).get("metrics"); - assertThat(allServicesMetrics.size(), is(5)); + assertEquals(5, allServicesMetrics.size()); } @Test @@ -192,9 +189,9 @@ public class RpcMetricsTest { tester.httpServer().setResponse(METRICS_RESPONSE); List<VespaService> services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - assertThat(services.size(), is(1)); + assertEquals(1, services.size()); Metrics metrics = services.get(0).getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); + assertEquals("Fetched number of metrics is not correct", 2, metrics.size()); Metric m = getMetric("foo.count", metrics); assertNotNull("Did not find expected metric with name 'foo.count'", m); @@ -203,7 +200,7 @@ public class RpcMetricsTest { try (RpcClient rpcClient = new RpcClient(tester.rpcPort())) { String response = getAllMetricNamesForService(services.get(0).getMonitoringName().id, vespaMetricsConsumerId, rpcClient); - assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); + assertEquals("foo.count=ON;output-name=foo_count,bar.count=OFF,", response); } } } @@ -263,11 +260,11 @@ public class RpcMetricsTest { } private static void verifyStatusMessage(JsonNode jsonObject) { - assertThat(jsonObject.get("status_code").intValue(), is(0)); - assertThat(jsonObject.get("status_msg").textValue(), notNullValue()); - assertThat(jsonObject.get("application").textValue(), notNullValue()); - assertThat(jsonObject.get("routing"), notNullValue()); - assertThat(jsonObject.size(), is(4)); + assertEquals(0, jsonObject.get("status_code").intValue()); + assertNotNull(jsonObject.get("status_msg")); + assertNotNull(jsonObject.get("application")); + assertNotNull(jsonObject.get("routing")); + assertEquals(4, jsonObject.size()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java index 6d009d2fd88..0ace697d545 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java @@ -6,8 +6,10 @@ import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + /** * @author Unknown @@ -29,35 +31,35 @@ public class ConfigSentinelClientTest { try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) { client.updateServiceStatuses(services); - assertThat(qrserver.getPid(), is(6520)); - assertThat(qrserver.getState(), is("RUNNING")); - assertThat(qrserver.isAlive(), is(true)); - assertThat(searchnode4.getPid(), is(6534)); - assertThat(searchnode4.getState(), is("RUNNING")); - assertThat(searchnode4.isAlive(), is(true)); + assertEquals(6520, qrserver.getPid()); + assertEquals("RUNNING", qrserver.getState()); + assertTrue(qrserver.isAlive()); + assertEquals(6534, searchnode4.getPid()); + assertEquals("RUNNING", searchnode4.getState()); + assertTrue(searchnode4.isAlive()); - assertThat(docproc.getPid(), is(-1)); - assertThat(docproc.getState(), is("FINISHED")); - assertThat(docproc.isAlive(), is(false)); + assertEquals(-1, docproc.getPid()); + assertEquals("FINISHED", docproc.getState()); + assertFalse(docproc.isAlive()); configsentinel.reConfigure(); client.ping(docproc); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); + assertEquals(100, docproc.getPid()); + assertEquals("RUNNING", docproc.getState()); + assertTrue(docproc.isAlive()); //qrserver has yet not been checked - assertThat(qrserver.isAlive(), is(true)); + assertTrue(qrserver.isAlive()); client.updateServiceStatuses(services); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); + assertEquals(100, docproc.getPid()); + assertEquals("RUNNING", docproc.getState()); + assertTrue(docproc.isAlive()); //qrserver is no longer running on this node - so should be false - assertThat(qrserver.isAlive(), is(false)); + assertFalse(qrserver.isAlive()); } } @@ -90,13 +92,13 @@ public class ConfigSentinelClientTest { try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) { client.updateServiceStatuses(services); - assertThat(container.isAlive(),is(true)); - assertThat(container.getPid(),is(14338)); - assertThat(container.getState(),is("RUNNING")); + assertTrue(container.isAlive()); + assertEquals(14338, container.getPid()); + assertEquals("RUNNING", container.getState()); - assertThat(containerClusterController.isAlive(),is(true)); - assertThat(containerClusterController.getPid(),is(25020)); - assertThat(containerClusterController.getState(),is("RUNNING")); + assertTrue(containerClusterController.isAlive()); + assertEquals(25020, containerClusterController.getPid()); + assertEquals("RUNNING", containerClusterController.getState()); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 44f0e9608ac..79dab3338e9 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -11,8 +11,8 @@ import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Unknown @@ -45,15 +45,15 @@ public class ContainerServiceTest { count++; System.out.println("Name: " + m.getName() + " value: " + m.getValue()); if (m.getDimensions().get(toDimensionId("chain")).equals("asvBlendingResult")) { - assertThat((Double)m.getValue(), is(26.4)); + assertEquals(26.4, m.getValue()); } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) { - assertThat((Double)m.getValue(), is(0.36666666666666664)); + assertEquals(0.36666666666666664, m.getValue()); } else { - assertThat("Unknown unknown chain", false, is(true)); + assertTrue("Unknown unknown chain", false); } } } - assertThat(count, is(2)); + assertEquals(2, count); } @After diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java index 3af317ab36b..5d4512276d4 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java @@ -4,27 +4,31 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.TestUtil; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.MetricId; import org.junit.Test; +import java.io.IOException; +import java.time.Instant; + import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** */ public class MetricsFetcherTest { - private static int port = 9; //port number is not used in this test + private static final int port = 9; //port number is not used in this test - private class MetricsConsumer implements MetricsParser.Consumer { + private static class MetricsConsumer implements MetricsParser.Consumer { Metrics metrics = new Metrics(); @Override public void consume(Metric metric) { metrics.add(metric); } } - Metrics fetch(String data) { + Metrics fetch(String data) throws IOException { RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); MetricsConsumer consumer = new MetricsConsumer(); fetcher.createMetrics(data, consumer, 0); @@ -32,30 +36,59 @@ public class MetricsFetcherTest { } @Test - public void testStateFormatMetricsParse() { + public void testStateFormatMetricsParse() throws IOException { String jsonData = TestUtil.getFileContents("metrics-state.json"); Metrics metrics = fetch(jsonData); - assertThat(metrics.size(), is(10)); - assertThat(getMetric("query_hits.count", metrics).getValue().intValue(), is(28)); - assertThat(getMetric("queries.rate", metrics).getValue().doubleValue(), is(0.4667)); - assertThat(metrics.getTimeStamp(), is(1334134700L)); + assertEquals(10, metrics.size()); + assertEquals(28L, getMetric("query_hits.count", metrics).getValue()); + assertEquals(0.4667, getMetric("queries.rate", metrics).getValue()); + assertEquals(Instant.ofEpochSecond(1334134700L), metrics.getTimeStamp()); } @Test - public void testEmptyJson() { + public void testEmptyJson() throws IOException { String jsonData = "{}"; Metrics metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + assertEquals(0, metrics.size()); + } + + @Test + public void testSkippingNullDimensions() throws IOException { + String jsonData = + "{\"status\" : {\"code\" : \"up\",\"message\" : \"Everything ok here\"}," + + "\"metrics\" : {\"snapshot\" : {\"from\" : 1334134640.089,\"to\" : 1334134700.088" + " }," + + "\"values\" : [" + + "{" + + " \"name\" : \"some.bogus.metric\"," + + " \"values\" : {" + + " \"count\" : 12," + + " \"rate\" : 0.2" + + " }," + + " \"dimensions\" : {" + + " \"version\" : null" + + " }" + + " }" + + "]}}"; + + Metrics metrics = fetch(jsonData); + assertEquals(2, metrics.size()); + assertTrue(metrics.list().get(0).getDimensions().isEmpty()); + assertTrue(metrics.list().get(1).getDimensions().isEmpty()); } @Test - public void testErrors() { + public void testErrors() throws IOException { String jsonData; - Metrics metrics; + Metrics metrics = null; jsonData = ""; - metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + try { + metrics = fetch(jsonData); + fail("Should have an IOException instead"); + } catch (IOException e) { + assertEquals("Expected start of object, got null", e.getMessage()); + } + assertNull(metrics); jsonData = "{\n" + "\"status\" : {\n" + @@ -64,7 +97,7 @@ public class MetricsFetcherTest { "}\n" + "}"; metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + assertEquals(0, metrics.size()); jsonData = "{\n" + "\"status\" : {\n" + @@ -92,8 +125,14 @@ public class MetricsFetcherTest { "}\n" + "}"; - metrics = fetch(jsonData); - assertThat("Wrong number of metrics", metrics.size(), is(0)); + metrics = null; + try { + metrics = fetch(jsonData); + fail("Should have an IOException instead"); + } catch (IllegalArgumentException e) { + assertEquals("Value for aggregator 'count' is not a number", e.getMessage()); + } + assertNull(metrics); } public Metric getMetric(String metric, Metrics metrics) { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java index 76909e050d5..30145746e79 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -188,7 +189,7 @@ public class SystemPollerTest { List<VespaService> services = List.of(s1); lastCpuJiffiesMetrics.put(s1, SystemPoller.getPidJiffies(new BufferedReader(new StringReader(perProcStats[0])))); - SystemPoller.JiffiesAndCpus next = SystemPoller.updateMetrics(prev, 1, + SystemPoller.JiffiesAndCpus next = SystemPoller.updateMetrics(prev, Instant.ofEpochSecond(1), new SystemPoller.GetJiffies() { @Override public SystemPoller.JiffiesAndCpus getTotalSystemJiffies() { diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java index a54e050bac7..6c0e1865dcd 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java @@ -3,7 +3,6 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.MetricId; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -11,9 +10,7 @@ import org.junit.Test; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -40,17 +37,17 @@ public class VespaServiceTest { @Test public void testService() { VespaService service = new VespaService("qrserver", "container/qrserver.0"); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); + assertEquals("qrserver", service.getServiceName()); + assertEquals("qrserver", service.getInstanceName()); + assertEquals(-1, service.getPid()); + assertEquals("container/qrserver.0", service.getConfigId()); service = VespaService.create("qrserver2", "container/qrserver.0", -1); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver2")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); + assertEquals("qrserver", service.getServiceName()); + assertEquals("qrserver2", service.getInstanceName()); + assertEquals(-1, service.getPid()); + assertEquals("container/qrserver.0", service.getConfigId()); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java index 264406a6fc6..dda8af8559b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java @@ -7,8 +7,7 @@ import org.junit.Test; import java.util.List; import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; /** * TODO: add more tests @@ -24,7 +23,7 @@ public class VespaServicesTest { new DummyService(1, "dummy/id/1")); VespaServices services = new VespaServices(dummyServices); - assertThat(services.getMonitoringServices("vespa.dummy").size(), is(2)); + assertEquals(2, services.getMonitoringServices("vespa.dummy").size()); } @Test @@ -33,7 +32,7 @@ public class VespaServicesTest { new DummyService(0, "dummy/id/0")); VespaServices services = new VespaServices(dummyServices); - assertThat(services.getMonitoringServices(ALL_SERVICES).size(), is(1)); + assertEquals(1, services.getMonitoringServices(ALL_SERVICES).size()); } } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 8fe5b2aecd1..b5808ebc699 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -86,12 +86,6 @@ <!-- Test --> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-junit</artifactId> - <version>2.0.0.0</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 6949f648865..cc780e277ad 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -263,9 +263,8 @@ public class UnixPath { return uncheck(() -> Files.deleteIfExists(path)); } - public UnixPath deleteIfExists() { - uncheck(() -> Files.deleteIfExists(path)); - return this; + public boolean deleteIfExists() { + return uncheck(() -> Files.deleteIfExists(path)); } /** @return false path does not exist, is not a directory, or has at least one entry. */ diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index 64406fcfafa..866b7d2877a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -21,9 +21,6 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; -import static org.hamcrest.junit.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -88,7 +85,7 @@ public class ConfigServerApiImplTest { @Test public void testBasicParsingSingleServer() { TestPojo answer = configServerApi.get("/path", TestPojo.class); - assertThat(answer.foo, is("bar")); + assertEquals(answer.foo, "bar"); assertLogStringContainsGETForAHost(); } @@ -139,8 +136,8 @@ public class ConfigServerApiImplTest { // ignore } - String[] log = mockLog.toString().split(" "); - assertThat(log, arrayContainingInAnyOrder("GET http://host1:666/path", "GET http://host2:666/path")); + List<String> log = List.of(mockLog.toString().split(" ")); + assertTrue(log.containsAll(List.of("GET http://host1:666/path", "GET http://host2:666/path"))); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java index 29581fe8b4d..8c914530122 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/flags/RealFlagRepositoryTest.java @@ -6,14 +6,13 @@ import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.flags.json.wire.WireFlagData; import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; -import org.hamcrest.collection.IsMapContaining; -import org.hamcrest.collection.IsMapWithSize; import org.junit.Test; import java.util.ArrayList; import java.util.Map; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -35,7 +34,7 @@ public class RealFlagRepositoryTest { when(configServerApi.get(any(), eq(WireFlagDataList.class))).thenReturn(list); Map<FlagId, FlagData> allFlagData = repository.getAllFlagData(); - assertThat(allFlagData, IsMapWithSize.aMapWithSize(1)); - assertThat(allFlagData, IsMapContaining.hasKey(new FlagId("id1"))); + assertEquals(1, allFlagData.size()); + assertTrue(allFlagData.containsKey(new FlagId("id1"))); } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 6a21814d5da..5d15d4353e2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -23,10 +23,8 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -103,13 +101,13 @@ public class RealNodeRepositoryTest { String dockerHostHostname = "dockerhost1.yahoo.com"; List<NodeSpec> containersToRun = nodeRepositoryApi.getNodes(dockerHostHostname); - assertThat(containersToRun.size(), is(1)); + assertEquals(1, containersToRun.size()); NodeSpec node = containersToRun.get(0); - assertThat(node.hostname(), is("host4.yahoo.com")); - assertThat(node.wantedDockerImage().get(), is(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"))); - assertThat(node.state(), is(NodeState.active)); - assertThat(node.wantedRestartGeneration().get(), is(0L)); - assertThat(node.currentRestartGeneration().get(), is(0L)); + assertEquals("host4.yahoo.com", node.hostname()); + assertEquals(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.42.0"), node.wantedDockerImage().get()); + assertEquals(NodeState.active, node.state()); + assertEquals(Long.valueOf(0), node.wantedRestartGeneration().get()); + assertEquals(Long.valueOf(0), node.currentRestartGeneration().get()); assertEquals(1, node.vcpu(), delta); assertEquals(4, node.memoryGb(), delta); assertEquals(100, node.diskGb(), delta); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java index 163c3410c4a..eea6c744bab 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTest.java @@ -6,9 +6,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.ChildProcessFailureEx import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import org.junit.Test; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -139,7 +137,7 @@ public class SystemCtlTest { systemCtl.serviceExists(taskContext, "foo"); fail(); } catch (Exception e) { - assertThat(e.getMessage(), containsString("garbage")); + assertTrue(e.getMessage().contains("garbage")); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java index 45e1dc73521..0ff98e2b5fe 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java @@ -5,10 +5,8 @@ import org.junit.Test; import java.util.Optional; -import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -174,7 +172,7 @@ public class YumPackageNameTest { YumPackageName.fromString("epoch:docker-engine-selinux-1.12.6-1.el7.x86_64"); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsStringIgnoringCase("epoch")); + assertTrue(e.getMessage().toLowerCase().contains("epoch")); } } diff --git a/parent/pom.xml b/parent/pom.xml index 2cebaf21833..1c24316a80b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -496,11 +496,6 @@ <version>${onnxruntime.version}</version> </dependency> <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - <version>0.6</version> - </dependency> - <dependency> <groupId>com.yahoo.athenz</groupId> <artifactId>athenz-zms-java-client</artifactId> <version>${athenz.version}</version> @@ -704,7 +699,7 @@ <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> - <version>1.8.4</version> + <version>1.9.3</version> </dependency> <dependency> <groupId>org.apache.velocity</groupId> @@ -870,7 +865,7 @@ <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> - <version>2.12.0</version> + <version>2.12.1</version> </dependency> </dependencies> </dependencyManagement> diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java index 1acfb3cdc7e..78db838cefe 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/FloatBucketResultNodeTestCase.java @@ -3,8 +3,9 @@ package com.yahoo.searchlib.expression; import org.junit.Test; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java index c038a207ba2..00c0739ed9a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeTestCase.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -25,8 +23,8 @@ public class IntegerBucketResultNodeTestCase extends ResultNodeTest { @Test public void testRange() { IntegerBucketResultNode bucket = new IntegerBucketResultNode(4, 10); - assertThat(bucket.getFrom(), is(4l)); - assertThat(bucket.getTo(), is(10l)); + assertEquals(4, bucket.getFrom()); + assertEquals(10, bucket.getTo()); assertFalse(bucket.empty()); assertTrue(dumpNode(bucket).contains("from: 4")); assertTrue(dumpNode(bucket).contains("to: 10")); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java index b081506dca8..744df472fb2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/IntegerResultNodeTestCase.java @@ -1,17 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; -import com.yahoo.vespa.objects.ObjectDumper; import org.junit.Test; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -29,52 +26,52 @@ public class IntegerResultNodeTestCase extends ResultNodeTest { @Test public void testClassId() { - assertThat(new Int8ResultNode().getClassId(), is(Int8ResultNode.classId)); - assertThat(new Int16ResultNode().getClassId(), is(Int16ResultNode.classId)); - assertThat(new Int32ResultNode().getClassId(), is(Int32ResultNode.classId)); - assertThat(new IntegerResultNode().getClassId(), is(IntegerResultNode.classId)); - assertThat(new BoolResultNode().getClassId(), is(BoolResultNode.classId)); + assertEquals(Int8ResultNode.classId, new Int8ResultNode().getClassId()); + assertEquals(Int16ResultNode.classId, new Int16ResultNode().getClassId()); + assertEquals(Int32ResultNode.classId, new Int32ResultNode().getClassId()); + assertEquals(IntegerResultNode.classId, new IntegerResultNode().getClassId()); + assertEquals(BoolResultNode.classId, new BoolResultNode().getClassId()); } @Test public void testTypeConversion() { for (NumericResultNode node : getResultNodes(3)) { - assertThat(node.getInteger(), is(3l)); - assertEquals(node.getFloat(), 3.0, 0.01); - assertThat(node.getRaw(), is(new byte[]{0, 0, 0, 0, 0, 0, 0, (byte) 3})); - assertThat(node.getString(), is("3")); - assertThat(node.getNumber().toString(), is("3")); + assertEquals(3, node.getInteger()); + assertEquals(3.0, node.getFloat(), 0.01); + assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, (byte) 3}, node.getRaw()); + assertEquals("3", node.getString()); + assertEquals("3", node.getNumber().toString()); } } @Test public void testMath() { for (NumericResultNode node : getResultNodes(5)) { - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); node.negate(); - assertThat(node.getInteger(), is(-5l)); + assertEquals(-5, node.getInteger()); node.multiply(new Int32ResultNode(3)); - assertThat(node.getInteger(), is(-15l)); + assertEquals(-15, node.getInteger()); node.add(new Int32ResultNode(1)); - assertThat(node.getInteger(), is(-14l)); + assertEquals(-14, node.getInteger()); node.divide(new Int32ResultNode(2)); - assertThat(node.getInteger(), is(-7l)); + assertEquals(-7, node.getInteger()); node.modulo(new Int32ResultNode(3)); - assertThat(node.getInteger(), is(-1l)); + assertEquals(-1, node.getInteger()); node.min(new Int32ResultNode(2)); - assertThat(node.getInteger(), is(-1l)); + assertEquals(-1, node.getInteger()); node.min(new Int32ResultNode(-2)); - assertThat(node.getInteger(), is(-2l)); + assertEquals(-2, node.getInteger()); node.max(new Int32ResultNode(-4)); - assertThat(node.getInteger(), is(-2l)); + assertEquals(-2, node.getInteger()); node.max(new Int32ResultNode(4)); - assertThat(node.getInteger(), is(4l)); - assertThat(node.onCmp(new Int32ResultNode(3)), is(1)); - assertThat(node.onCmp(new Int32ResultNode(4)), is(0)); - assertThat(node.onCmp(new Int32ResultNode(5)), is(-1)); + assertEquals(4, node.getInteger()); + assertEquals(1, node.onCmp(new Int32ResultNode(3))); + assertEquals(0, node.onCmp(new Int32ResultNode(4))); + assertEquals(-1, node.onCmp(new Int32ResultNode(5))); node.set(new Int32ResultNode(8)); - assertThat(node.getInteger(), is(8l)); - assertThat(node.hashCode(), is((int)(8 + node.getClassId()))); + assertEquals(8, node.getInteger()); + assertEquals(8 + node.getClassId(), node.hashCode()); assertTrue(dumpNode(node).contains("value: 8")); } } @@ -95,37 +92,37 @@ public class IntegerResultNodeTestCase extends ResultNodeTest { public void testInt8() { Int8ResultNode node = new Int8ResultNode(); node.setValue((byte) 5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testInt16() { Int16ResultNode node = new Int16ResultNode(); node.setValue((short)5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testInt32() { Int32ResultNode node = new Int32ResultNode(); node.setValue(5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testLong() { IntegerResultNode node = new IntegerResultNode(); node.setValue(5); - assertThat(node.getInteger(), is(5l)); + assertEquals(5, node.getInteger()); } @Test public void testSerialization() throws IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { for (NumericResultNode node : getResultNodes(8)) { - assertThat(node.getInteger(), is(8L)); + assertEquals(8, node.getInteger()); NumericResultNode out = node.getClass().getConstructor().newInstance(); assertCorrectSerialization(node, out); - assertThat(out.getInteger(), is(node.getInteger())); + assertEquals(out.getInteger(), node.getInteger()); } } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java index ff2af418ad4..64c30769f60 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/NullResultNodeTestCase.java @@ -3,13 +3,8 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.ObjectDumper; import org.junit.Test; - -import java.util.regex.Pattern; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -20,17 +15,17 @@ public class NullResultNodeTestCase { @Test public void testNullResultNode() { NullResultNode nullRes = new NullResultNode(); - assertThat(nullRes.onGetClassId(), is(NullResultNode.classId)); - assertThat(nullRes.getInteger(), is(0l)); - assertThat(nullRes.getString(), is("")); - assertThat(nullRes.getRaw(), is(new byte[0])); - assertEquals(nullRes.getFloat(), 0.0, 0.01); - assertThat(nullRes.onCmp(new NullResultNode()), is(0)); - assertThat(nullRes.onCmp(new IntegerResultNode(0)), is(not(0))); + assertEquals(NullResultNode.classId, nullRes.onGetClassId()); + assertEquals(0, nullRes.getInteger()); + assertTrue(nullRes.getString().isEmpty()); + assertEquals(0, nullRes.getRaw().length); + assertEquals(0.0, nullRes.getFloat(), 0.01); + assertEquals(0, nullRes.onCmp(new NullResultNode())); + assertNotEquals(0, nullRes.onCmp(new IntegerResultNode(0))); ObjectDumper dumper = new ObjectDumper(); nullRes.visitMembers(dumper); assertTrue(dumper.toString().contains("result: <NULL>")); nullRes.set(new IntegerResultNode(3)); - assertThat(nullRes.onCmp(new IntegerResultNode(3)), is(not(0))); + assertNotEquals(0, nullRes.onCmp(new IntegerResultNode(3))); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java index eef4f225778..ab9fb2e2e50 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/RawBucketResultNodeTestCase.java @@ -3,9 +3,8 @@ package com.yahoo.searchlib.expression; import org.junit.Test; -import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -24,8 +23,8 @@ public class RawBucketResultNodeTestCase extends ResultNodeTest { public void testRange() { RawBucketResultNode bucket = new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), new RawResultNode(new byte[]{9, 6})); assertFalse(bucket.empty()); - assertThat(bucket.getFrom(), is(new byte[]{6, 9})); - assertThat(bucket.getTo(), is(new byte[]{9, 6})); + assertArrayEquals(new byte[]{6, 9}, bucket.getFrom()); + assertArrayEquals(new byte[]{9, 6}, bucket.getTo()); assertCorrectSerialization(bucket, new RawBucketResultNode()); assertTrue(dumpNode(bucket).contains("value: RawData(data = [6, 9])")); assertTrue(dumpNode(bucket).contains("value: RawData(data = [9, 6])")); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java index fb033801352..52be4d422d5 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeTest.java @@ -4,8 +4,7 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.ObjectDumper; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -24,7 +23,7 @@ public class ResultNodeTest { from.serialize(buffer); buffer.flip(); to.deserialize(buffer); - assertThat(from.onCmp(to), is(0)); + assertEquals(0, from.onCmp(to)); } public void assertOrder(ResultNode a, ResultNode b, ResultNode c) { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java index 54f4744897a..614e6bb608a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ResultNodeVectorTestCase.java @@ -1,14 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -import com.yahoo.vespa.objects.BufferSerializer; import org.junit.Test; - -import java.util.List; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -18,12 +13,12 @@ import static org.junit.Assert.assertTrue; public class ResultNodeVectorTestCase extends ResultNodeTest { @Test public void testClassId() { - assertThat(new IntegerResultNodeVector().getClassId(), is(IntegerResultNodeVector.classId)); - assertThat(new Int32ResultNodeVector().getClassId(), is(Int32ResultNodeVector.classId)); - assertThat(new Int16ResultNodeVector().getClassId(), is(Int16ResultNodeVector.classId)); - assertThat(new Int8ResultNodeVector().getClassId(), is(Int8ResultNodeVector.classId)); - assertThat(new FloatResultNodeVector().getClassId(), is(FloatResultNodeVector.classId)); - assertThat(new BoolResultNodeVector().getClassId(), is(BoolResultNodeVector.classId)); + assertEquals(IntegerResultNodeVector.classId, new IntegerResultNodeVector().getClassId()); + assertEquals(Int32ResultNodeVector.classId, new Int32ResultNodeVector().getClassId()); + assertEquals(Int16ResultNodeVector.classId, new Int16ResultNodeVector().getClassId()); + assertEquals(Int8ResultNodeVector.classId, new Int8ResultNodeVector().getClassId()); + assertEquals(FloatResultNodeVector.classId, new FloatResultNodeVector().getClassId()); + assertEquals(BoolResultNodeVector.classId, new BoolResultNodeVector().getClassId()); } @Test @@ -32,37 +27,37 @@ public class ResultNodeVectorTestCase extends ResultNodeTest { b.add(new BoolResultNode(true)); b.add(new BoolResultNode(false)); b.add((ResultNode)new BoolResultNode(false)); - assertThat(b.getVector().size(), is(3)); + assertEquals(3, b.getVector().size()); Int8ResultNodeVector i8 = new Int8ResultNodeVector(); i8.add(new Int8ResultNode((byte)9)); i8.add(new Int8ResultNode((byte)2)); i8.add((ResultNode)new Int8ResultNode((byte)5)); - assertThat(i8.getVector().size(), is(3)); + assertEquals(3, i8.getVector().size()); Int16ResultNodeVector i16 = new Int16ResultNodeVector(); i16.add(new Int16ResultNode((short)9)); i16.add(new Int16ResultNode((short)2)); i16.add((ResultNode)new Int16ResultNode((short)5)); - assertThat(i16.getVector().size(), is(3)); + assertEquals(3, i16.getVector().size()); Int32ResultNodeVector i32 = new Int32ResultNodeVector(); i32.add(new Int32ResultNode(9)); i32.add(new Int32ResultNode(2)); i32.add((ResultNode)new Int32ResultNode(5)); - assertThat(i32.getVector().size(), is(3)); + assertEquals(3, i32.getVector().size()); IntegerResultNodeVector ieger = new IntegerResultNodeVector(); ieger.add(new IntegerResultNode(9)); ieger.add(new IntegerResultNode(2)); ieger.add((ResultNode)new IntegerResultNode(5)); - assertThat(ieger.getVector().size(), is(3)); + assertEquals(3, ieger.getVector().size()); FloatResultNodeVector floatvec = new FloatResultNodeVector(); floatvec.add(new FloatResultNode(3.3)); floatvec.add(new FloatResultNode(3.4)); floatvec.add((ResultNode)new FloatResultNode(3.5)); - assertThat(floatvec.getVector().size(), is(3)); + assertEquals(3, floatvec.getVector().size()); } @Test @@ -155,11 +150,11 @@ public class ResultNodeVectorTestCase extends ResultNodeTest { } private void assertVecEqual(ResultNodeVector vec1, ResultNodeVector vec2) { - assertThat(vec1.onCmp(vec2), is(0)); + assertEquals(0, vec1.onCmp(vec2)); } private void assertClassCmp(ResultNodeVector add) { - assertThat(add.onCmp(new NullResultNode()), is(not(0))); + assertNotEquals(0, add.onCmp(new NullResultNode())); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java index 15c89b4beb3..fee60c9df7b 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/StringBucketResultNodeTestCase.java @@ -2,9 +2,7 @@ package com.yahoo.searchlib.expression; import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** @@ -22,8 +20,8 @@ public class StringBucketResultNodeTestCase extends ResultNodeTest { @Test public void testRange() { StringBucketResultNode bucket = new StringBucketResultNode("a", "d"); - assertThat(bucket.getFrom(), is("a")); - assertThat(bucket.getTo(), is("d")); + assertEquals("a", bucket.getFrom()); + assertEquals("d", bucket.getTo()); assertTrue(dumpNode(bucket).contains("value: 'a'")); assertTrue(dumpNode(bucket).contains("value: 'd'")); assertCorrectSerialization(bucket, new StringBucketResultNode()); diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index f1fb58eb9d0..271497398a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -25,32 +25,33 @@ struct ChangeBase { DIV, CLEARDOC }; - enum {UNSET_ENUM = 0xffffffffu}; + enum {UNSET_ENTRY_REF = 0xffffffffu}; ChangeBase() : _type(NOOP), _doc(0), _weight(1), - _enumScratchPad(UNSET_ENUM) + _cached_entry_ref(UNSET_ENTRY_REF) { } ChangeBase(Type type, uint32_t d, int32_t w = 1) : _type(type), _doc(d), _weight(w), - _enumScratchPad(UNSET_ENUM) + _cached_entry_ref(UNSET_ENTRY_REF) { } int cmp(const ChangeBase &b) const { int diff(_doc - b._doc); return diff; } bool operator <(const ChangeBase & b) const { return cmp(b) < 0; } - uint32_t getEnum() const { return _enumScratchPad; } - void setEnum(uint32_t value) const { _enumScratchPad = value; } - bool isEnumValid() const { return _enumScratchPad != UNSET_ENUM; } + uint32_t get_entry_ref() const { return _cached_entry_ref; } + void set_entry_ref(uint32_t entry_ref) const { _cached_entry_ref = entry_ref; } + bool has_entry_ref() const { return _cached_entry_ref != UNSET_ENTRY_REF; } + void clear_entry_ref() const { _cached_entry_ref = UNSET_ENTRY_REF; } Type _type; uint32_t _doc; int32_t _weight; - mutable uint32_t _enumScratchPad; + mutable uint32_t _cached_entry_ref; }; template <typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index 8136e654152..f0ff23a06b4 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -56,6 +56,7 @@ protected: virtual void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) = 0; vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const override; void populate_address_space_usage(AddressSpaceUsage& usage) const override; + void cache_change_data_entry_ref(const Change& c) const; public: EnumAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); ~EnumAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 3e578856c2b..c0680fd9238 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -84,6 +84,15 @@ EnumAttribute<B>::populate_address_space_usage(AddressSpaceUsage& usage) const usage.set(AddressSpaceComponents::enum_store, _enumStore.get_values_address_space_usage()); } +template <typename B> +void +EnumAttribute<B>::cache_change_data_entry_ref(const Change& c) const +{ + EnumIndex new_idx; + _enumStore.find_index(c._data.raw(), new_idx); + c.set_entry_ref(new_idx.ref()); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 251bbd7c8a7..7df30b895d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -25,10 +25,10 @@ template <typename B, typename M> bool MultiValueEnumAttribute<B, M>::extractChangeData(const Change & c, EnumIndex & idx) { - if ( ! c.isEnumValid() ) { + if ( ! c.has_entry_ref() ) { return this->_enumStore.find_index(c._data.raw(), idx); } - idx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + idx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); return true; } @@ -42,9 +42,9 @@ MultiValueEnumAttribute<B, M>::considerAttributeChange(const Change & c, EnumSto { EnumIndex idx; if (!this->_enumStore.find_index(c._data.raw(), idx)) { - c.setEnum(inserter.insert(c._data.raw()).ref()); + c.set_entry_ref(inserter.insert(c._data.raw()).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } } } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index dde853cbc90..a51c9804cf2 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -140,9 +140,9 @@ SingleValueEnumAttribute<B>::considerUpdateAttributeChange(const Change & c, Enu { EnumIndex idx; if (!this->_enumStore.find_index(c._data.raw(), idx)) { - c.setEnum(inserter.insert(c._data.raw()).ref()); + c.set_entry_ref(inserter.insert(c._data.raw()).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } considerUpdateAttributeChange(c); // for numeric } @@ -168,8 +168,8 @@ SingleValueEnumAttribute<B>::applyUpdateValueChange(const Change& c, EnumStoreBa { EnumIndex oldIdx = _enumIndices[c._doc]; EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { this->_enumStore.find_index(c._data.raw(), newIdx); } @@ -181,6 +181,8 @@ void SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) { ValueModifier valueGuard(this->getValueModifier()); + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, updater); @@ -192,6 +194,8 @@ SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) applyUpdateValueChange(clearDoc, updater); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index d6622f7c880..c6b720ba14e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -38,9 +38,9 @@ SingleValueNumericEnumAttribute<B>::considerArithmeticAttributeChange(const Chan EnumIndex idx; if (!this->_enumStore.find_index(newValue, idx)) { - c.setEnum(inserter.insert(newValue).ref()); + c.set_entry_ref(inserter.insert(newValue).ref()); } else { - c.setEnum(idx.ref()); + c.set_entry_ref(idx.ref()); } _currDocValues[c._doc] = newValue; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 1083d0f4cb8..4d37171e151 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -47,8 +47,8 @@ SingleValueNumericPostingAttribute<B>::applyUpdateValueChange(const Change & c, std::map<DocId, EnumIndex> & currEnumIndices) { EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { enumStore.find_index(c._data.raw(), newIdx); } @@ -89,6 +89,8 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -114,6 +116,8 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& applyUpdateValueChange(clearDoc, enumStore, currEnumIndices); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_comparator(), currEnumIndices, changePost); diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index e77c59e915d..4670ee075fe 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -45,8 +45,8 @@ SingleValueStringPostingAttributeT<B>::applyUpdateValueChange(const Change & c, std::map<DocId, EnumIndex> &currEnumIndices) { EnumIndex newIdx; - if (c.isEnumValid()) { - newIdx = EnumIndex(vespalib::datastore::EntryRef(c.getEnum())); + if (c.has_entry_ref()) { + newIdx = EnumIndex(vespalib::datastore::EntryRef(c.get_entry_ref())); } else { enumStore.find_index(c._data.raw(), newIdx); } @@ -90,6 +90,8 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; + // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. + this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -105,6 +107,8 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& applyUpdateValueChange(this->_defaultValue, enumStore, currEnumIndices); } } + // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. + this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_folded_comparator(), dictionary, currEnumIndices, changePost); diff --git a/security-utils/pom.xml b/security-utils/pom.xml index 39a52fb12db..55b4b190f97 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -40,11 +40,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> diff --git a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java index b0d587bb19a..f6f48d8b1b8 100644 --- a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java @@ -7,10 +7,9 @@ import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -55,8 +54,8 @@ public class KeyUtilsTest { public void can_serialize_and_deserialize_rsa_publickey_using_pem_format() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); String pem = KeyUtils.toPem(keyPair.getPublic()); - assertThat(pem, containsString("BEGIN PUBLIC KEY")); - assertThat(pem, containsString("END PUBLIC KEY")); + assertTrue(pem.contains("BEGIN PUBLIC KEY")); + assertTrue(pem.contains("END PUBLIC KEY")); PublicKey deserializedKey = KeyUtils.fromPemEncodedPublicKey(pem); assertEquals(keyPair.getPublic(), deserializedKey); assertEquals(KeyAlgorithm.RSA.getAlgorithmName(), deserializedKey.getAlgorithm()); @@ -66,8 +65,8 @@ public class KeyUtilsTest { public void can_serialize_and_deserialize_ec_publickey_using_pem_format() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); String pem = KeyUtils.toPem(keyPair.getPublic()); - assertThat(pem, containsString("BEGIN PUBLIC KEY")); - assertThat(pem, containsString("END PUBLIC KEY")); + assertTrue(pem.contains("BEGIN PUBLIC KEY")); + assertTrue(pem.contains("END PUBLIC KEY")); PublicKey deserializedKey = KeyUtils.fromPemEncodedPublicKey(pem); assertEquals(keyPair.getPublic(), deserializedKey); assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm()); @@ -76,8 +75,8 @@ public class KeyUtilsTest { private static void testPrivateKeySerialization(KeyAlgorithm keyAlgorithm, KeyFormat keyFormat, String pemLabel) { KeyPair keyPair = KeyUtils.generateKeypair(keyAlgorithm); String pem = KeyUtils.toPem(keyPair.getPrivate(), keyFormat); - assertThat(pem, containsString("BEGIN " + pemLabel)); - assertThat(pem, containsString("END " + pemLabel)); + assertTrue(pem.contains("BEGIN " + pemLabel)); + assertTrue(pem.contains("END " + pemLabel)); PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); assertEquals(keyPair.getPrivate(), deserializedKey); assertEquals(keyAlgorithm.getAlgorithmName(), deserializedKey.getAlgorithm()); diff --git a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java index ff0076b9cc4..32b8dfc5bcd 100644 --- a/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/Pkcs10CsrUtilsTest.java @@ -6,9 +6,8 @@ import org.junit.Test; import javax.security.auth.x500.X500Principal; import java.security.KeyPair; -import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -22,8 +21,8 @@ public class Pkcs10CsrUtilsTest { Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(subject, keypair, SignatureAlgorithm.SHA512_WITH_ECDSA).build(); String pem = Pkcs10CsrUtils.toPem(csr); Pkcs10Csr deserializedCsr = Pkcs10CsrUtils.fromPem(pem); - assertThat(pem, containsString("BEGIN CERTIFICATE REQUEST")); - assertThat(pem, containsString("END CERTIFICATE REQUEST")); + assertTrue(pem.contains("BEGIN CERTIFICATE REQUEST")); + assertTrue(pem.contains("END CERTIFICATE REQUEST")); assertEquals(subject, deserializedCsr.getSubject()); } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 7075179234e..b2e800542b8 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -13,12 +13,9 @@ import java.util.Arrays; import java.util.List; import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.is; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -32,8 +29,8 @@ public class X509CertificateUtilsTest { X509Certificate cert = TestUtils.createCertificate(keypair, subject); assertEquals(subject, cert.getSubjectX500Principal()); String pem = X509CertificateUtils.toPem(cert); - assertThat(pem, containsString("BEGIN CERTIFICATE")); - assertThat(pem, containsString("END CERTIFICATE")); + assertTrue(pem.contains("BEGIN CERTIFICATE")); + assertTrue(pem.contains("END CERTIFICATE")); X509Certificate deserializedCert = X509CertificateUtils.fromPem(pem); assertEquals(subject, deserializedCert.getSubjectX500Principal()); } @@ -70,8 +67,8 @@ public class X509CertificateUtilsTest { .build(); List<SubjectAlternativeName> sans = X509CertificateUtils.getSubjectAlternativeNames(cert); - assertThat(sans.size(), is(1)); - assertThat(sans.get(0), equalTo(san)); + assertEquals(1, sans.size()); + assertEquals(san, sans.get(0)); } @Test diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java index d7be4ead078..13dc7ad4624 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/HttpRequestStrategyTest.java @@ -205,7 +205,9 @@ class HttpRequestStrategyTest { DocumentId id2 = DocumentId.of("ns", "type", "2"); DocumentId id3 = DocumentId.of("ns", "type", "3"); DocumentId id4 = DocumentId.of("ns", "type", "4"); + DocumentId id5 = DocumentId.of("ns", "type", "5"); HttpRequest failing = new HttpRequest("POST", "/", null, null, null); + HttpRequest partial = new HttpRequest("POST", "/", null, null, null); HttpRequest request = new HttpRequest("POST", "/", null, null, null); HttpRequest blocking = new HttpRequest("POST", "/", null, null, null); @@ -218,22 +220,28 @@ class HttpRequestStrategyTest { blocker.arriveAndAwaitAdvance(); // ... block dispatch thread, so we get something in the queue. throw new RuntimeException("armageddon"); // Dispatch thread should die, tearing down everything. } - else if (req == failing) { + else if (req == partial) { phaser.arriveAndAwaitAdvance(); // Let test thread enqueue more ops before failing (and retrying) this. vessel.completeExceptionally(new IOException("failed")); - phaser.arriveAndAwaitAdvance(); // Ensure a retry is scheduled before test thread is allowed to continue. } - else phaser.arriveAndAwaitAdvance(); // Don't complete from mock cluster, but require destruction to do this. + else if (req == failing) { + System.err.println("failing"); + vessel.completeExceptionally(new RuntimeException("fatal")); + } }); + // inflight completes dispatch, but causes no response. CompletableFuture<HttpResponse> inflight = strategy.enqueue(id1, request); + // serialised 1 and 2 are waiting for the above inflight to complete. CompletableFuture<HttpResponse> serialised1 = strategy.enqueue(id1, request); CompletableFuture<HttpResponse> serialised2 = strategy.enqueue(id1, request); - CompletableFuture<HttpResponse> failed = strategy.enqueue(id2, failing); - CompletableFuture<HttpResponse> blocked = strategy.enqueue(id3, blocking); - CompletableFuture<HttpResponse> delayed = strategy.enqueue(id4, request); - phaser.arriveAndAwaitAdvance(); // inflight completes dispatch, but causes no response. - phaser.arriveAndAwaitAdvance(); // failed is allowed to dispatch ... - phaser.arriveAndAwaitAdvance(); // ... and a retry is enqueued. + CompletableFuture<HttpResponse> retried = strategy.enqueue(id2, partial); + CompletableFuture<HttpResponse> failed = strategy.enqueue(id3, failing); + CompletableFuture<HttpResponse> blocked = strategy.enqueue(id4, blocking); + CompletableFuture<HttpResponse> delayed = strategy.enqueue(id5, request); + phaser.arriveAndAwaitAdvance(); // retried is allowed to dispatch, and will be retried async. + // failed immediately fails, and lets us assert the above retry is indeed enqueued. + assertEquals("ai.vespa.feed.client.FeedException: java.lang.RuntimeException: fatal", + assertThrows(ExecutionException.class, failed::get).getMessage()); phaser.arriveAndAwaitAdvance(); // blocked starts dispatch, and hangs, blocking dispatch thread. // Current state: inflight is "inflight to cluster", serialised1/2 are waiting completion of it; @@ -242,7 +250,8 @@ class HttpRequestStrategyTest { assertFalse(inflight.isDone()); assertFalse(serialised1.isDone()); assertFalse(serialised2.isDone()); - assertFalse(failed.isDone()); + assertTrue(failed.isDone()); + assertFalse(retried.isDone()); assertFalse(blocked.isDone()); assertFalse(delayed.isDone()); @@ -259,7 +268,7 @@ class HttpRequestStrategyTest { assertEquals("ai.vespa.feed.client.FeedException: Operation aborted", assertThrows(ExecutionException.class, delayed::get).getMessage()); assertEquals("ai.vespa.feed.client.FeedException: java.io.IOException: failed", - assertThrows(ExecutionException.class, failed::get).getMessage()); + assertThrows(ExecutionException.class, retried::get).getMessage()); assertEquals("ai.vespa.feed.client.FeedException: Operation aborted", assertThrows(ExecutionException.class, strategy.enqueue(id1, request)::get).getMessage()); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java index 01784226c9e..6f1b5eebcc4 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java @@ -10,8 +10,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPOutputStream; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -42,7 +41,7 @@ public class FeedHandlerCompressionTest { } processedInput.append((char)readValue); } - assertThat(processedInput.toString(), is(testData)); + assertEquals(processedInput.toString(), testData); } /** @@ -64,7 +63,7 @@ public class FeedHandlerCompressionTest { } processedInput.append((char)readValue); } - assertThat(processedInput.toString(), is(testData)); + assertEquals(processedInput.toString(), testData); } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 1fd6425f945..2f4afb0c2a5 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -30,9 +30,8 @@ import java.io.InputStream; import java.util.concurrent.Executor; import java.util.concurrent.Executors; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,8 +46,8 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1)); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); - assertThat(Utf8.toString(outStream.toByteArray()), is("1230 OK message trace\n")); + assertEquals(httpResponse.getContentType(), "text/plain"); + assertEquals(Utf8.toString(outStream.toByteArray()), "1230 OK message trace\n"); } @Test @@ -57,9 +56,9 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createBrokenRequest()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); - assertThat(Utf8.toString(outStream.toByteArray()), startsWith("1230 ERROR ")); - assertThat(metric.get(MetricNames.PARSE_ERROR), is(1L)); + assertEquals(httpResponse.getContentType(), "text/plain"); + assertTrue(Utf8.toString(outStream.toByteArray()).startsWith("1230 ERROR ")); + assertEquals(1L, metric.get(MetricNames.PARSE_ERROR)); } @Test @@ -68,9 +67,9 @@ public class FeedHandlerV3Test { HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100)); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); + assertEquals(httpResponse.getContentType(), "text/plain"); String result = Utf8.toString(outStream.toByteArray()); - assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); + assertEquals(101, Splitter.on("\n").splitToList(result).size()); } private static DocumentTypeManager createDoctypeManager() { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java index a13f4f79ca9..6858c4bede3 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -10,10 +10,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> @@ -34,86 +33,86 @@ public class VersionsTestCase { private static final List<String> GARBAGE = Collections.singletonList("garbage"); @Test - public void testEmpty() throws Exception { + public void testEmpty() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(EMPTY); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testOneTwo() throws Exception { + public void testOneTwo() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testOneThree() throws Exception { + public void testOneThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testTwoThree() throws Exception { + public void testTwoThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneNullThree() throws Exception { + public void testOneNullThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneCommaThree() throws Exception { + public void testOneCommaThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testOneEmptyThree() throws Exception { + public void testOneEmptyThree() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_THREE); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test public void testTooLarge() throws Exception { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TOO_LARGE_NUMBER); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertTrue(v.first instanceof ErrorHttpResponse); ByteArrayOutputStream errorMsg = new ByteArrayOutputStream(); ErrorHttpResponse errorResponse = (ErrorHttpResponse) v.first; errorResponse.render(errorMsg); - assertThat(errorMsg.toString(), - is("Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + - "Server supports protocol versions [3]")); - assertThat(v.second, is(-1)); + assertEquals(errorMsg.toString(), + "Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + + "Server supports protocol versions [3]"); + assertEquals(Integer.valueOf(-1), v.second); } @Test - public void testThreeTooLarge() throws Exception { + public void testThreeTooLarge() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_TOO_LARGE_NUMBER); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testTwoCommaTooLarge() throws Exception { + public void testTwoCommaTooLarge() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_COMMA_TOO_LARGE_NUMBER); - assertThat(v.first, nullValue()); - assertThat(v.second, is(3)); + assertNull(v.first); + assertEquals(Integer.valueOf(3), v.second); } @Test - public void testGarbage() throws Exception { + public void testGarbage() { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(GARBAGE); - assertThat(v.first, instanceOf(ErrorHttpResponse.class)); - assertThat(v.second, is(-1)); + assertTrue(v.first instanceof ErrorHttpResponse); + assertEquals(Integer.valueOf(-1), v.second); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 5b23d5d92ae..dfbcb06c365 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.collect.ImmutableList; import com.yahoo.text.Ascii7BitMatcher; import java.util.ArrayList; @@ -86,7 +85,7 @@ public class TensorType { private final Value valueType; /** Sorted list of the dimensions of this */ - private final ImmutableList<Dimension> dimensions; + private final List<Dimension> dimensions; private final TensorType mappedSubtype; @@ -94,7 +93,7 @@ public class TensorType { this.valueType = valueType; List<Dimension> dimensionList = new ArrayList<>(dimensions); Collections.sort(dimensionList); - this.dimensions = ImmutableList.copyOf(dimensionList); + this.dimensions = List.copyOf(dimensionList); if (dimensionList.stream().allMatch(d -> d.isIndexed())) mappedSubtype = empty; diff --git a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java b/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java index 349ba8998bd..1d5b6a6cac7 100644 --- a/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java +++ b/yolean/src/test/java/com/yahoo/yolean/chain/ChainBuilderTest.java @@ -10,7 +10,6 @@ import java.util.List; import static com.yahoo.yolean.chain.Dependencies.after; import static com.yahoo.yolean.chain.Dependencies.before; import static com.yahoo.yolean.chain.Dependencies.provides; -import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; |